Repository: dbt-labs/dbt-codegen
Branch: main
Commit: c4d495b8bb27
Files: 80
Total size: 137.5 KB
Directory structure:
gitextract_phy8nu37/
├── .circleci/
│ └── config.yml
├── .github/
│ ├── CODEOWNERS
│ ├── ISSUE_TEMPLATE/
│ │ ├── bug_report.md
│ │ └── feature_request.md
│ ├── pull_request_template.md
│ └── workflows/
│ ├── ci.yml
│ ├── stale.yml
│ └── triage-labels.yml
├── .gitignore
├── CHANGELOG.md
├── CONTRIBUTING.md
├── LICENSE
├── Makefile
├── README.md
├── RELEASE.md
├── bash_scripts/
│ └── base_model_creation.sh
├── dbt_project.yml
├── docker-compose.yml
├── integration_tests/
│ ├── README.md
│ ├── dbt_project.yml
│ ├── macros/
│ │ ├── assert_equal.sql
│ │ ├── integer_type_value.sql
│ │ ├── operations/
│ │ │ └── create_source_table.sql
│ │ └── text_type_value.sql
│ ├── models/
│ │ ├── child_model.sql
│ │ ├── model_data_a.sql
│ │ ├── model_from_source.sql
│ │ ├── model_incremental.sql
│ │ ├── model_repeated.sql
│ │ ├── model_struct.sql
│ │ ├── model_without_any_ctes.sql
│ │ ├── model_without_import_ctes.sql
│ │ ├── schema.yml
│ │ └── source.yml
│ ├── package-lock.yml
│ ├── packages.yml
│ ├── profiles.yml
│ ├── seeds/
│ │ ├── data__a_relation.csv
│ │ ├── data__b_relation.csv
│ │ └── data__campaign_analytics.csv
│ └── tests/
│ ├── test_generate_base_models.sql
│ ├── test_generate_base_models_all_args.sql
│ ├── test_generate_base_models_case_sensitive.sql
│ ├── test_generate_base_models_leading.sql
│ ├── test_generate_model_import_ctes.sql
│ ├── test_generate_model_import_ctes_leading.sql
│ ├── test_generate_model_import_ctes_no_ctes.sql
│ ├── test_generate_model_repeated_yaml.sql
│ ├── test_generate_model_struct_yaml.sql
│ ├── test_generate_model_yaml.sql
│ ├── test_generate_model_yaml_multiple_models.sql
│ ├── test_generate_model_yaml_upstream_descriptions.sql
│ ├── test_generate_model_yaml_upstream_source_descriptions.sql
│ ├── test_generate_source.sql
│ ├── test_generate_source_all_args.sql
│ ├── test_generate_source_exclude.sql
│ ├── test_generate_source_include_database_property.sql
│ ├── test_generate_source_include_schema_property.sql
│ ├── test_generate_source_some_tables.sql
│ ├── test_generate_source_table_descriptions.sql
│ ├── test_generate_source_table_name.sql
│ ├── test_generate_source_table_pattern.sql
│ ├── test_generate_unit_test_template.sql
│ ├── test_generate_unit_test_template_incremental.sql
│ ├── test_generate_unit_test_template_inline_columns.sql
│ ├── test_generate_unit_test_template_model_from_source.sql
│ ├── test_generate_unit_test_template_no_inputs.sql
│ └── test_helper_get_models.sql
├── macros/
│ ├── create_base_models.sql
│ ├── generate_base_model.sql
│ ├── generate_model_import_ctes.sql
│ ├── generate_model_yaml.sql
│ ├── generate_source.sql
│ ├── generate_unit_test_template.sql
│ ├── helpers/
│ │ └── helpers.sql
│ └── vendored/
│ └── dbt_core/
│ └── format_column.sql
├── packages.yml
├── run_test.sh
├── supported_adapters.env
└── tox.ini
================================================
FILE CONTENTS
================================================
================================================
FILE: .circleci/config.yml
================================================
version: 2
jobs:
integration-postgres:
docker:
- image: cimg/python:3.11
- image: cimg/postgres:17.0
environment:
POSTGRES_USER: root
environment:
POSTGRES_HOST: localhost
POSTGRES_USER: root
DBT_ENV_SECRET_POSTGRES_PASS: ''
POSTGRES_PORT: 5432
POSTGRES_DATABASE: circle_test
POSTGRES_SCHEMA: codegen_integration_tests_postgres
steps:
- checkout
- run: pip install dbt-core dbt-postgres
- run:
name: "Run Tests - Postgres"
command: |
cd integration_tests
dbt --warn-error deps --target postgres
dbt --warn-error run-operation create_source_table --target postgres
dbt --warn-error seed --target postgres --full-refresh
dbt --warn-error run --target postgres
dbt --warn-error test --target postgres
- store_artifacts:
path: integration_tests/logs
- store_artifacts:
path: integration_tests/target
# The resource_class feature allows configuring CPU and RAM resources for each job. Different resource classes are available for different executors. https://circleci.com/docs/2.0/configuration-reference/#resourceclass
resource_class: large
integration-redshift:
docker:
- image: cimg/python:3.11
steps:
- checkout
- run: pip install dbt-core dbt-redshift
- run:
name: "Run Tests - Redshift"
command: |
cd integration_tests
dbt --warn-error deps --target redshift
dbt --warn-error run-operation create_source_table --target redshift
dbt --warn-error seed --target redshift --full-refresh
dbt --warn-error run --target redshift
dbt --warn-error test --target redshift
- store_artifacts:
path: integration_tests/logs
- store_artifacts:
path: integration_tests/target
# The resource_class feature allows configuring CPU and RAM resources for each job. Different resource classes are available for different executors. https://circleci.com/docs/2.0/configuration-reference/#resourceclass
resource_class: large
integration-snowflake:
docker:
- image: cimg/python:3.11
steps:
- checkout
- run: pip install dbt-core dbt-snowflake
- run:
name: "Run Tests - Snowflake"
command: |
cd integration_tests
dbt --warn-error deps --target snowflake
dbt --warn-error run-operation create_source_table --target snowflake
dbt --warn-error seed --target snowflake --full-refresh
dbt --warn-error run --target snowflake
dbt --warn-error test --target snowflake
- store_artifacts:
path: integration_tests/logs
- store_artifacts:
path: integration_tests/target
# The resource_class feature allows configuring CPU and RAM resources for each job. Different resource classes are available for different executors. https://circleci.com/docs/2.0/configuration-reference/#resourceclass
resource_class: large
integration-bigquery:
environment:
BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json"
docker:
- image: cimg/python:3.11
steps:
- checkout
- run: pip install dbt-core dbt-bigquery
- run:
name: Setup Environment Variables
command: |
echo $BIGQUERY_SERVICE_ACCOUNT_JSON > $BIGQUERY_SERVICE_KEY_PATH
echo 'export BIGQUERY_KEYFILE_JSON="$BIGQUERY_SERVICE_ACCOUNT_JSON"' >> "$BASH_ENV"
- run:
name: "Run Tests - BigQuery"
command: |
cd integration_tests
dbt --warn-error deps --target bigquery
dbt --warn-error run-operation create_source_table --target bigquery
dbt --warn-error seed --target bigquery --full-refresh
dbt --warn-error run --target bigquery
dbt --warn-error test --target bigquery
- store_artifacts:
path: integration_tests/logs
- store_artifacts:
path: integration_tests/target
# The resource_class feature allows configuring CPU and RAM resources for each job. Different resource classes are available for different executors. https://circleci.com/docs/2.0/configuration-reference/#resourceclass
resource_class: large
workflows:
version: 2
test-all:
jobs:
- integration-postgres:
context: profile-postgres
- integration-redshift:
context: profile-redshift
requires:
- integration-postgres
- integration-snowflake:
context: profile-snowflake
requires:
- integration-postgres
- integration-bigquery:
context: profile-bigquery
requires:
- integration-postgres
================================================
FILE: .github/CODEOWNERS
================================================
* @dbt-labs/dbt-package-owners
================================================
FILE: .github/ISSUE_TEMPLATE/bug_report.md
================================================
---
name: Bug report
about: Report a bug or an issue you've found with this package
title: ''
labels: bug, triage
assignees: ''
---
### Describe the bug
<!---
A clear and concise description of what the bug is. You can also use the issue title to do this
--->
### Steps to reproduce
<!---
In as much detail as possible, please provide steps to reproduce the issue. Sample data that triggers the issue, example model code, etc is all very helpful here.
--->
### Expected results
<!---
A clear and concise description of what you expected to happen.
--->
### Actual results
<!---
A clear and concise description of what you expected to happen.
--->
### Screenshots and log output
<!---
If applicable, add screenshots or log output to help explain your problem.
--->
### System information
**The contents of your `packages.yml` file:**
**Which database are you using dbt with?**
- [ ] postgres
- [ ] redshift
- [ ] bigquery
- [ ] snowflake
- [ ] other (specify: ____________)
**The output of `dbt --version`:**
```
<output goes here>
```
**The operating system you're using:**
**The output of `python --version`:**
### Additional context
<!---
Add any other context about the problem here. For example, if you think you know which line of code is causing the issue.
--->
### Are you interested in contributing the fix?
<!---
Let us know if you want to contribute the fix, and whether would need a hand getting started
--->
================================================
FILE: .github/ISSUE_TEMPLATE/feature_request.md
================================================
---
name: Feature request
about: Suggest an idea for this package
title: ''
labels: enhancement, triage
assignees: ''
---
### Describe the feature
A clear and concise description of what you want to happen.
### Describe alternatives you've considered
A clear and concise description of any alternative solutions or features you've considered.
### Additional context
Is this feature database-specific? Which database(s) is/are relevant? Please include any other relevant context here.
### Who will this benefit?
What kind of use case will this feature be useful for? Please be specific and provide examples, this will help us prioritize properly.
### Are you interested in contributing this feature?
<!---
Let us know if you want to contribute the feature, and whether would need a hand getting started
--->
================================================
FILE: .github/pull_request_template.md
================================================
resolves #
### Problem
<!---
Describe the problem this PR is solving. What is the application state
before this PR is merged?
-->
### Solution
<!---
Describe the way this PR solves the above problem. Add as much detail as you
can to help reviewers understand your changes. Include any alternatives and
tradeoffs you considered.
-->
## Checklist
- [ ] This code is associated with an [issue](https://github.com/dbt-labs/dbt-codegen/issues) which has been triaged and [accepted for development](https://docs.getdbt.com/docs/contributing/oss-expectations#pull-requests).
- [ ] I have read [the contributing guide](https://github.com/dbt-labs/dbt-codegen/blob/main/CONTRIBUTING.md) and understand what's expected of me
- [ ] I have run this code in development and it appears to resolve the stated issue
- [ ] This PR includes tests, or tests are not required/relevant for this PR
- [ ] I have updated the README.md (if applicable)
================================================
FILE: .github/workflows/ci.yml
================================================
# **what?**
# Run tests for dbt-codegen against supported adapters
# **why?**
# To ensure that dbt-codegen works as expected with all supported adapters
# **when?**
# On every PR, and every push to main and when manually triggered
name: Package Integration Tests
on:
push:
branches:
- main
pull_request_target:
workflow_dispatch:
jobs:
run-tests:
uses: dbt-labs/dbt-package-testing/.github/workflows/run_tox.yml@v1
# this just tests with postgres so no variables need to be passed through.
# When it's time to add more adapters you will need to pass through inputs for
# the other adapters as shown in the below example for redshift
with:
# snowflake
SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }}
SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE }}
SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }}
SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }}
SNOWFLAKE_SCHEMA: "integration_tests_snowflake_${{ github.run_number }}"
# bigquery
BIGQUERY_PROJECT: ${{ vars.BIGQUERY_PROJECT }}
BIGQUERY_SCHEMA: "integration_tests_bigquery_${{ github.run_number }}"
# redshift
REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }}
REDSHIFT_USER: ${{ vars.REDSHIFT_USER }}
REDSHIFT_DATABASE: ${{ vars.REDSHIFT_DATABASE }}
REDSHIFT_SCHEMA: "integration_tests_redshift_${{ github.run_number }}"
REDSHIFT_PORT: ${{ vars.REDSHIFT_PORT }}
secrets:
SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.SNOWFLAKE_PASS }}
DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.REDSHIFT_PASS }}
BIGQUERY_KEYFILE_JSON: ${{ secrets.BIGQUERY_KEYFILE_JSON }}
================================================
FILE: .github/workflows/stale.yml
================================================
# **what?**
# For issues that have been open for awhile without activity, label
# them as stale with a warning that they will be closed out. If
# anyone comments to keep the issue open, it will automatically
# remove the stale label and keep it open.
# Stale label rules:
# awaiting_response, more_information_needed -> 90 days
# good_first_issue, help_wanted -> 360 days (a year)
# tech_debt -> 720 (2 years)
# all else defaults -> 180 days (6 months)
# **why?**
# To keep the repo in a clean state from issues that aren't relevant anymore
# **when?**
# Once a day
name: "Close stale issues and PRs"
on:
schedule:
- cron: "30 1 * * *"
permissions:
issues: write
pull-requests: write
jobs:
stale:
uses: dbt-labs/actions/.github/workflows/stale-bot-matrix.yml@main
================================================
FILE: .github/workflows/triage-labels.yml
================================================
# **what?**
# When the maintenance team triages, we sometimes need more information from the issue creator. In
# those cases we remove the `triage` label and add the `awaiting_response` label. Once we
# recieve a response in the form of a comment, we want the `awaiting_response` label removed
# in favor of the `triage` label so we are aware that the issue needs action.
# **why?**
# To help with out team triage issue tracking
# **when?**
# This will run when a comment is added to an issue and that issue has the `awaiting_response` label.
name: Update Triage Label
on: issue_comment
defaults:
run:
shell: bash
permissions:
issues: write
jobs:
triage_label:
if: contains(github.event.issue.labels.*.name, 'awaiting_response')
uses: dbt-labs/actions/.github/workflows/swap-labels.yml@main
with:
add_label: "triage"
remove_label: "awaiting_response"
secrets: inherit
================================================
FILE: .gitignore
================================================
target/
dbt_modules/
dbt_packages/
logs/
env*/
.venv/
.env/
venv/
================================================
FILE: CHANGELOG.md
================================================
# dbt-codegen v0.13.1
## What's Changed
## Under the hood
* Temporarily remove CI test for case-sensitive identifiers when generating sources by @dbeatty10 in https://github.com/dbt-labs/dbt-codegen/pull/230
**Full Changelog**: https://github.com/dbt-labs/dbt-codegen/compare/0.13.0...0.13.1
# dbt-codegen v0.13.0
## What's Changed
### Features
* Read upstream descriptions from sources by @esegal in https://github.com/dbt-labs/dbt-codegen/pull/154
* Parameters in `generate_source` for case-sensitive identifiers by @pnadolny13 in https://github.com/dbt-labs/dbt-codegen/pull/168
### Fixes
* Escape upstream descriptions in generate_model_yaml by @wircho in https://github.com/dbt-labs/dbt-codegen/pull/159
* Fix quoted identifiers in the `generate_base_model` macro for BigQuery by @dbeatty10 in https://github.com/dbt-labs/dbt-codegen/pull/199
### Docs
* fix generate_source example by @yatsky in https://github.com/dbt-labs/dbt-codegen/pull/164
* Improve developer README by @gwenwindflower in https://github.com/dbt-labs/dbt-codegen/pull/163
* Fix bad spacing in dev README by @gwenwindflower in https://github.com/dbt-labs/dbt-codegen/pull/170
* Changelogs for 0.12.0, 0.12.1, and 0.13.0-b1 by @dbeatty10 in https://github.com/dbt-labs/dbt-codegen/pull/196
## Under the hood
* Restore CI test for case-sensitive identifiers when generating sources by @dbeatty10 in https://github.com/dbt-labs/dbt-codegen/pull/192
* Remove Redshift-specific logic for toggling case-sensitive identifiers by @dbeatty10 in https://github.com/dbt-labs/dbt-codegen/pull/208
* Use the `cimg/postgres` Docker image by @dbeatty10 in https://github.com/dbt-labs/dbt-codegen/pull/214
* Independent CircleCI workflow job for each tested adapter by @dbeatty10 in https://github.com/dbt-labs/dbt-codegen/pull/215
* Simplify environment variables for BigQuery in CircleCI by @dbeatty10 in https://github.com/dbt-labs/dbt-codegen/pull/216
* Stop installing prereleases from PyPI in favor of stable releases only by @dbeatty10 in https://github.com/dbt-labs/dbt-codegen/pull/220
* Upgrade to Python 3.11 in CircleCI by @dbeatty10 in https://github.com/dbt-labs/dbt-codegen/pull/222
* Use dynamic schema names rather than hardcoded ones by @dbeatty10 in https://github.com/dbt-labs/dbt-codegen/pull/224
* Add support for postgres testing in GitHub CI via tox by @emmyoop by @emmyoop in https://github.com/dbt-labs/dbt-codegen/pull/181
* Add support for snowflake testing in GitHub CI via tox by @emmyoop in https://github.com/dbt-labs/dbt-codegen/pull/198
* Add support for redshift testing in GitHub CI via tox by @emmyoop in https://github.com/dbt-labs/dbt-codegen/pull/204
* Add support for bigquery testing in GitHub CI via tox by @emmyoop in https://github.com/dbt-labs/dbt-codegen/pull/203
## New Contributors
* @wircho made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/159
* @esegal made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/154
* @yatsky made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/164
* @gwenwindflower made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/163
* @pnadolny13 made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/168
* @emmyoop made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/181
**Full Changelog**: https://github.com/dbt-labs/dbt-codegen/compare/0.12.1...0.13.0
# dbt-codegen v0.13.0-b1
## What's Changed
### Features
* Read upstream descriptions from sources by @esegal in https://github.com/dbt-labs/dbt-codegen/pull/154
* Case sensitive generate source by @pnadolny13 in https://github.com/dbt-labs/dbt-codegen/pull/168
### Fixes
* Escape upstream descriptions in generate_model_yaml by @wircho in https://github.com/dbt-labs/dbt-codegen/pull/159
### Docs
* fix generate_source example by @yatsky in https://github.com/dbt-labs/dbt-codegen/pull/164
* Improve developer README by @gwenwindflower in https://github.com/dbt-labs/dbt-codegen/pull/163
* Fix bad spacing in dev README by @gwenwindflower in https://github.com/dbt-labs/dbt-codegen/pull/170
* Update Changelog by @gwenwindflower in https://github.com/dbt-labs/dbt-codegen/pull/174
## New Contributors
- @wircho made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/159
- @yatsky made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/164
- @pnadolny13 made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/168
- @esegal made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/154
- @gwenwindflower made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/163
**Full Changelog**: https://github.com/dbt-labs/dbt-codegen/compare/0.12.1...v0.13.0-b1
# dbt-codegen v0.12.1
## What's Changed
* Add dispatch to macros by @jeremyyeo in https://github.com/dbt-labs/dbt-codegen/pull/148
* Remove terminal output in the generated file. by @vijmen in https://github.com/dbt-labs/dbt-codegen/pull/149
## New Contributors
* @jeremyyeo made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/148
* @vijmen made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/149
**Full Changelog**: https://github.com/dbt-labs/dbt-codegen/compare/0.12.0...0.12.1
# dbt-codegen v0.12.0
## What's Changed
* Use print for outputting codegen by @JorgenG in https://github.com/dbt-labs/dbt-codegen/pull/86
## New Contributors
* @JorgenG made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/86
**Full Changelog**: https://github.com/dbt-labs/dbt-codegen/compare/0.11.0...0.12.0
# dbt-codegen v0.11.0
## 🚨 Breaking change
`include_data_types` parameter added to `generate_model_yaml` and behavior changed for `generate_source`. Both default to `true`
and are lowercase to align with the dbt style guide. Scale & precision are **not** included. Previous logic for `generate_source` defaulted to `false` and the resulting data types were uppercase and included scale & precision ([#122](https://github.com/dbt-labs/dbt-codegen/pull/122)).
[Dispatch](https://docs.getdbt.com/reference/dbt-jinja-functions/dispatch) can be used to utilize the column data type formatting of previous versions. Namely, by adding this macro to your project:
```sql
{% macro default__data_type_format_source(column) %}
{{ return(column.data_type | upper) }}
{% endmacro %}
```
And then adding this within `dbt_project.yml`:
```yaml
dispatch:
- macro_namespace: codegen
search_order: ["my_project", "codegen"]
```
## What's Changed
- GitHub Action to add/remove triage labels as-needed by @dbeatty10 in https://github.com/dbt-labs/dbt-codegen/pull/133
- GitHub Action to close issues as stale as-needed by @dbeatty10 in https://github.com/dbt-labs/dbt-codegen/pull/134
- Update README.md by @cohms in https://github.com/dbt-labs/dbt-codegen/pull/129
- Remove hard-coded values for database and schema by @dbeatty10 in https://github.com/dbt-labs/dbt-codegen/pull/139
- Instructions for the release process by @dbeatty10 in https://github.com/dbt-labs/dbt-codegen/pull/137
- Add `include_data_types` argument to `generate_model_yaml` macro by @linbug in https://github.com/dbt-labs/dbt-codegen/pull/122
## New Contributors
- @cohms made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/129
- @linbug made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/122
**Full Changelog**: https://github.com/dbt-labs/dbt-codegen/compare/0.10.0...v0.10.0
# dbt-codegen v0.10.0
## What's Changed
- added comments to verbose regex in generate_model_import_ctes by @graciegoheen in https://github.com/dbt-labs/dbt-codegen/pull/93
- Feature/hackathon model generator by @fivetran-joemarkiewicz in https://github.com/dbt-labs/dbt-codegen/pull/83
- Suggestion to include packages.yml example in README.md by @Maayan-s in https://github.com/dbt-labs/dbt-codegen/pull/77
- Add include_data_types flag to generate_source macro by @GSokol in https://github.com/dbt-labs/dbt-codegen/pull/76
- Expected result of nested struct in BigQuery by @dbeatty10 in https://github.com/dbt-labs/dbt-codegen/pull/105
- issue106/get_models helper macro by @erkanncelen in https://github.com/dbt-labs/dbt-codegen/pull/115
- Feat/generate sources add database and schema by @jeremyholtzman in https://github.com/dbt-labs/dbt-codegen/pull/124
## New Contributors
- @fivetran-joemarkiewicz made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/83
- @Maayan-s made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/77
- @GSokol made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/76
- @erkanncelen made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/115
- @jeremyholtzman made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/124
**Full Changelog**: https://github.com/dbt-labs/dbt-codegen/compare/0.9.0...0.10.0
# dbt-codegen v0.9.0
# dbt-codegen v0.8.1
# dbt-codegen v0.8.0
# Unreleased
## Breaking changes
## New features
## Quality of life
- Now uses `print` instead of `log` to output the generated text into the console. This enables you to invoke dbt with the `--quiet` flag and directly pipe the codegen output into a new file, ending up with valid yaml
## Under the hood
## Contributors:
- [@JorgenG](https://github.com/JorgenG) (#86)
# dbt-codegen v0.7.0
## 🚨 Breaking change
- Add support for including description placeholders for the source and table, which changes the behavior of `generate_source` when `include_descriptions` is set to `True`. Previous logic only created description placeholders for the columns ([#64](https://github.com/dbt-labs/dbt-codegen/issues/64), [#66](https://github.com/dbt-labs/dbt-codegen/pull/66))
## New features
- Add optional `table_names` arg to `generate_source` ([#50](https://github.com/dbt-labs/dbt-codegen/issues/50), [#51](https://github.com/dbt-labs/dbt-codegen/pull/51))
- Add support for importing descriptions from columns with the same names in upstream models. It is available by setting the parameter `upstream_descriptions` to `True` in `generate_model_yaml` ([#61](https://github.com/dbt-labs/dbt-codegen/pull/61))
- Added `case_sensitive_cols` argument to `generate_base_model` macro ([#63](https://github.com/dbt-labs/dbt-codegen/pull/63))
- Add optional `name` arg to `generate_source` ([#64](https://github.com/dbt-labs/dbt-codegen/issues/64), [#66](https://github.com/dbt-labs/dbt-codegen/pull/66))
## Fixes
- `generate_model_yaml` now correctly handles nested `STRUCT` fields in BigQuery ([#27](https://github.com/dbt-labs/dbt-codegen/issues/27), [#54](https://github.com/dbt-labs/dbt-codegen/pull/54))
## Contributors:
- [@rahulj51](https://github.com/rahulj51) (#51)
- [@bodschut](https://github.com/bodschut) (#54)
- [@b-per](https://github.com/b-per) (#61)
- [@graciegoheen](https://github.com/graciegoheen) (#63)
- [@kbrock91](https://github.com/kbrock91) (#66)
# dbt-codegen v0.6.0
This release creates breaking changes to the `generate_source.sql` macro.
## Features
- add optional `table_pattern` argument to `generate_source.sql` macro. Default value is '%' to pull all tables in the raw data schema to preserve existing behavior if the `table_pattern` argument is not specified by the user.
# dbt-codegen v0.5.0
This release supports any version (minor and patch) of v1, which means far less need for compatibility releases in the future.
## Under the hood
- Change `require-dbt-version` to `[">=1.0.0", "<2.0.0"]`
- Bump dbt-utils dependency
- Replace `source-paths` and `data-paths` with `model-paths` and `seed-paths` respectively
- Rename `data` and `analysis` directories to `seeds` and `analyses` respectively
- Replace `dbt_modules` with `dbt_packages` in `clean-targets`
# dbt-codegen v0.4.1
🚨 This is a compatibility release in preparation for `dbt-core` v1.0.0 (🎉). Projects using this version with `dbt-core` v1.0.x can expect to see a deprecation warning. This will be resolved in the next minor release.
# dbt-codegen v0.4.0
## Breaking changes
- Requires `dbt>=0.20.0` and `dbt-utils>=0.7.0`
- Depends on `dbt-labs/dbt_utils` (instead of `fishtown-analytics/dbt_utils`)
## Features
- Add optional `leading_commas` arg to `generate_base_model` (#41 @jaypeedevlin)
- Add optional `include_descriptions` arg to `generate_source` (#40 @djbelknapdbs)
## Fixes
- In the `generate_source` macro, use `dbt_utils.get_relations_by_pattern` instead of `get_relations_by_prefix`, since the latter will be deprecated in the future (#42)
## Under the hood
- Use new adapter.dispatch syntax (#44)
# dbt-codegen v0.3.2
This is a quality of life release
## Other
- Fix rendering issues on hub.getdbt.com
- Fix integration tests due to python version compatibility
# dbt-codegen v0.3.1
This is a bugfix release
## Fixes
- Use latest version of dbt-utils (0.6.2) to ensure generate_source_yaml works for non-target schemata (#34)
# dbt-codegen v0.3.0
## 🚨 Breaking change
This release requires dbt v0.18.0, and dbt-utils v0.6.1. If you're not ready to upgrade, consider using a previous release of this package.
## Quality of life
- Use dbt v0.18.0 (#31)
- Fix README rendering on hub (#32 @calvingiles)
# dbt-codegen v0.2.0
## 🚨 Breaking change
The lower bound of `dbt-utils` is now `0.4.0`.
This won't affect most users, since you're likely already using version of dbt-utils higher than this to achieve 0.17.0 compatibility.
## Quality of life:
- Change dbt-utils dependencies to `[>=0.4.0, <0.6.0]` (#29)
- Fix tests (#29)
# dbt-codegen v0.1.0
## 🚨 Breaking change!
This package now requires dbt v0.17.x!
## Features:
- Add `generate_model_yaml` (#18 @jtalmi)
## Under the hood:
- Update to v0.17.0, including `dbt_project.yml` version 2 syntax (#23)
- Add GitHub templates and installation instructions (#23)
## Acknowledgements
@marzaccaro made a PR for `generate_model_yaml`, and, although I had reviewed it, I let the PR go stale and somehow completely forgot about it when merging PR #18 — this is completely my bad! So equal credit to @marzaccaro and @jtalmi for their work :clap:
# dbt-codegen v0.0.4
This is a bugfix release to improve compatibility with Snowflake
# dbt-codegen v0.0.3
Bump utils version range
# dbt-codegen v0.0.2
Small quality of life improvements
# dbt-codegen v0.0.1
Initial release
================================================
FILE: CONTRIBUTING.md
================================================
# Contributing to `dbt-codegen`
`dbt-codegen` is open source software. It is what it is today because community members have opened issues, provided feedback, and [contributed to the knowledge loop](https://www.getdbt.com/dbt-labs/values/). Whether you are a seasoned open source contributor or a first-time committer, we welcome and encourage you to contribute code, documentation, ideas, or problem statements to this project.
Remember: all PRs (apart from cosmetic fixes like typos) should be [associated with an issue](https://docs.getdbt.com/docs/contributing/oss-expectations#pull-requests).
1. [About this document](#about-this-document)
1. [Getting the code](#getting-the-code)
1. [Setting up an environment](#setting-up-an-environment)
1. [Implementation guidelines](#implementation-guidelines)
1. [Testing dbt-codegen](#testing)
1. [Adding CHANGELOG Entry](#adding-changelog-entry)
1. [Submitting a Pull Request](#submitting-a-pull-request)
## About this document
There are many ways to contribute to the ongoing development of `dbt-codegen`, such as by participating in discussions and issues. We encourage you to first read our higher-level document: ["Expectations for Open Source Contributors"](https://docs.getdbt.com/docs/contributing/oss-expectations).
The rest of this document serves as a more granular guide for contributing code changes to `dbt-codegen` (this repository). It is not intended as a guide for using `dbt-codegen`, and some pieces assume a level of familiarity with Python development (virtualenvs, `pip`, etc). Specific code snippets in this guide assume you are using macOS or Linux and are comfortable with the command line.
### Notes
- **CLA:** Please note that anyone contributing code to `dbt-codegen` must sign the [Contributor License Agreement](https://docs.getdbt.com/docs/contributor-license-agreements). If you are unable to sign the CLA, the `dbt-codegen` maintainers will unfortunately be unable to merge any of your Pull Requests. We welcome you to participate in discussions, open issues, and comment on existing ones.
- **Branches:** All pull requests from community contributors should target the `main` branch (default). If the change is needed as a patch for a version of `dbt-codegen` that has already been released (or is already a release candidate), a maintainer will backport the changes in your PR to the relevant branch.
## Getting the code
### Installing git
You will need `git` in order to download and modify the `dbt-codegen` source code. On macOS, the best way to download git is to just install [Xcode](https://developer.apple.com/support/xcode/).
### External contributors
If you are not a member of the `dbt-labs` GitHub organization, you can contribute to `dbt-codegen` by forking the `dbt-codegen` repository. For a detailed overview on forking, check out the [GitHub docs on forking](https://help.github.com/en/articles/fork-a-repo). In short, you will need to:
1. Fork the `dbt-codegen` repository
2. Clone your fork locally
3. Check out a new branch for your proposed changes
4. Push changes to your fork
5. Open a pull request against `dbt-labs/dbt-codegen` from your forked repository
### dbt Labs contributors
If you are a member of the `dbt-labs` GitHub organization, you will have push access to the `dbt-codegen` repo. Rather than forking `dbt-codegen` to make your changes, just clone the repository, check out a new branch, and push directly to that branch.
## Setting up an environment
There are some tools that will be helpful to you in developing locally. While this is the list relevant for `dbt-codegen` development, many of these tools are used commonly across open-source python projects.
### Tools
These are the tools used in `dbt-codegen` development and testing:
- [`make`](https://users.cs.duke.edu/~ola/courses/programming/Makefiles/Makefiles.html) to run multiple setup or test steps in combination. Don't worry too much, nobody _really_ understands how `make` works, and our Makefile aims to be super simple.
- [CircleCI](https://circleci.com/) for automating tests and checks, once a PR is pushed to the `dbt-codegen` repository
A deep understanding of these tools in not required to effectively contribute to `dbt-codegen`, but we recommend checking out the attached documentation if you're interested in learning more about each one.
## Testing
Once you're able to manually test that your code change is working as expected, it's important to run existing automated tests, as well as adding some new ones. These tests will ensure that:
- Your code changes do not unexpectedly break other established functionality
- Your code changes can handle all known edge cases
- The functionality you're adding will _keep_ working in the future
See here for details for running existing integration tests and adding new ones:
- [integration_tests/README.md](integration_tests/README.md)
## Adding CHANGELOG Entry
We use [automatically generated release notes](https://docs.github.com/en/repositories/releasing-projects-on-github/automatically-generated-release-notes) to generate `CHANGELOG` entries. **Note:** Do not edit the `CHANGELOG.md` directly. Your modifications will be lost.
You don't need to worry about which `dbt-codegen` version your change will go into. Just create the changelog entry at the top of CHANGELOG.md and open your PR against the `main` branch. All merged changes will be included in the next minor version of `dbt-codegen`. The maintainers _may_ choose to "backport" specific changes in order to patch older minor versions. In that case, a maintainer will take care of that backport after merging your PR, before releasing the new version of `dbt-codegen`.
## Submitting a Pull Request
A `dbt-codegen` maintainer will review your PR. They may suggest code revision for style or clarity, or request that you add unit or integration test(s). These are good things! We believe that, with a little bit of help, anyone can contribute high-quality code.
Automated tests run via CircleCI. If you're a first-time contributor, all tests (including code checks and unit tests) will require a maintainer to approve. Changes in the `dbt-codegen` repository trigger integration tests.
Once all tests are passing and your PR has been approved, a `dbt-codegen` maintainer will merge your changes into the active development branch. And that's it! Happy developing :tada:
================================================
FILE: LICENSE
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: Makefile
================================================
.DEFAULT_GOAL:=help
.PHONY: test
test: ## Run the integration tests.
@./run_test.sh $(target)
.PHONY: test_tox
test: ## Run the integration tests with tox
@\
tox -e dbt_integration_$(target)
.PHONY: dev
dev: ## Installs dbt-* packages in develop mode along with development dependencies.
@\
echo "Install dbt-$(target)..."; \
python -m pip install --upgrade pip setuptools; \
python -m pip install dbt-core "dbt-$(target)";
.PHONY: setup-db
setup-db: ## Setup Postgres database with docker-compose for system testing.
@\
docker-compose up --detach postgres
.PHONY: help
help: ## Show this help message.
@echo 'usage: make [target]'
@echo
@echo 'targets:'
@grep -E '^[8+a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
================================================
FILE: README.md
================================================
# dbt-codegen
Macros that generate dbt code, and log it to the command line.
# Contents
- [dbt-codegen](#dbt-codegen)
- [Contents](#contents)
- [Installation instructions](#installation-instructions)
- [Macros](#macros)
- [generate_source (source)](#generate_source-source)
- [Arguments](#arguments)
- [Usage:](#usage)
- [generate_base_model (source)](#generate_base_model-source)
- [Arguments:](#arguments-1)
- [Usage:](#usage-1)
- [create_base_models (source)](#create_base_models-source)
- [Arguments:](#arguments-2)
- [Usage:](#usage-2)
- [base_model_creation (source)](#base_model_creation-source)
- [Arguments:](#arguments-3)
- [Usage:](#usage-3)
- [generate_model_yaml (source)](#generate_model_yaml-source)
- [Arguments:](#arguments-4)
- [Usage:](#usage-4)
- [generate_model_import_ctes (source)](#generate_model_import_ctes-source)
- [Arguments:](#arguments-5)
- [Usage:](#usage-5)
- [generate_unit_test_template (source)](#generate_unit_test_template-source)
- [Arguments:](#arguments-6)
- [Usage:](#usage-6)
- [Contributing](#contributing)
# Installation instructions
New to dbt packages? Read more about them [here](https://docs.getdbt.com/docs/building-a-dbt-project/package-management/).
1. Include this package in your `packages.yml` file — check [here](https://hub.getdbt.com/dbt-labs/codegen/latest/) for the latest version number:
```yml
packages:
- package: dbt-labs/codegen
version: X.X.X ## update to latest version here
```
2. Run `dbt deps` to install the package.
# Macros
## generate_source ([source](macros/generate_source.sql))
This macro generates lightweight YAML for a [Source](https://docs.getdbt.com/docs/using-sources),
which you can then paste into a schema file.
### Arguments
- `schema_name` (required): The schema name that contains your source data
- `database_name` (optional, default=target.database): The database that your
source data is in.
- `table_names` (optional, default=none): A list of tables that you want to generate the source definitions for.
- `generate_columns` (optional, default=False): Whether you want to add the
column names to your source definition.
- `include_descriptions` (optional, default=False): Whether you want to add
description placeholders to your source definition.
- `include_data_types` (optional, default=True): Whether you want to add data
types to your source columns definitions.
- `table_pattern` (optional, default='%'): A table prefix / postfix that you
want to subselect from all available tables within a given schema.
- `exclude` (optional, default=''): A string you want to exclude from the selection criteria
- `name` (optional, default=schema_name): The name of your source
- `include_database` (optional, default=False): Whether you want to add
the database to your source definition
- `include_schema` (optional, default=False): Whether you want to add
the schema to your source definition
- `case_sensitive_databases` (optional, default=False): Whether you want database names to be
in lowercase, or to match the case in the source table — not compatible with Redshift
- `case_sensitive_schemas` (optional, default=False): Whether you want schema names to be
in lowercase, or to match the case in the source table — not compatible with Redshift
- `case_sensitive_tables` (optional, default=False): Whether you want table names to be
in lowercase, or to match the case in the source table — not compatible with Redshift
- `case_sensitive_cols` (optional, default=False): Whether you want column names to be
in lowercase, or to match the case in the source table
### Outputting to a file
If you use the `dbt run-operation` approach it is possible to output directly to a file by piping the output to a new file and using the `--quiet` CLI flag:
```
dbt --quiet run-operation generate_source --args '{"table_names": ["orders"]}' > models/staging/jaffle_shop/_sources.yml
```
### Usage:
1. Copy the macro into a statement tab in the dbt Cloud IDE, or into an analysis file, and compile your code
```
{{ codegen.generate_source('raw_jaffle_shop') }}
```
or for multiple arguments
```
{{ codegen.generate_source(schema_name= 'jaffle_shop', database_name= 'raw') }}
```
Alternatively, call the macro as an [operation](https://docs.getdbt.com/docs/using-operations):
```
$ dbt run-operation generate_source --args 'schema_name: raw_jaffle_shop'
```
or
```
# for multiple arguments, use the dict syntax
$ dbt run-operation generate_source --args '{"schema_name": "jaffle_shop", "database_name": "raw", "table_names":["table_1", "table_2"]}'
```
or if you want to include column names and data types:
```
$ dbt run-operation generate_source --args '{"schema_name": "jaffle_shop", "generate_columns": true}'
```
or if you want to include column names without data types (the behavior dbt-codegen <= v0.9.0):
```
$ dbt run-operation generate_source --args '{"schema_name": "jaffle_shop", "generate_columns": true, "include_data_types": false}'
```
2. The YAML for the source will be logged to the command line
```
version: 2
sources:
- name: raw_jaffle_shop
database: raw
schema: raw_jaffle_shop
tables:
- name: customers
description: ""
- name: orders
description: ""
- name: payments
description: ""
```
3. Paste the output in to a schema `.yml` file, and refactor as required.
## generate_base_model ([source](macros/generate_base_model.sql))
This macro generates the SQL for a base model, which you can then paste into a
model.
### Arguments:
- `source_name` (required): The source you wish to generate base model SQL for.
- `table_name` (required): The source table you wish to generate base model SQL for.
- `leading_commas` (optional, default=False): Whether you want your commas to be leading (vs trailing).
- `case_sensitive_cols ` (optional, default=False): Whether your source table has case sensitive column names. If true, keeps the case of the column names from the source.
- `materialized` (optional, default=None): Set materialization style (e.g. table, view, incremental) inside of the model's `config` block. If not set, materialization style will be controlled by `dbt_project.yml`
### Usage:
1. Create a source for the table you wish to create a base model on top of.
2. Copy the macro into a statement tab in the dbt Cloud IDE, or into an analysis file, and compile your code
```
{{ codegen.generate_base_model(
source_name='raw_jaffle_shop',
table_name='customers',
materialized='table'
) }}
```
Alternatively, call the macro as an [operation](https://docs.getdbt.com/docs/using-operations):
```
$ dbt run-operation generate_base_model --args '{"source_name": "raw_jaffle_shop", "table_name": "customers"}'
```
3. The SQL for a base model will be logged to the command line
```
with source as (
select * from {{ source('raw_jaffle_shop', 'customers') }}
),
renamed as (
select
id,
first_name,
last_name,
email,
_elt_updated_at
from source
)
select * from renamed
```
4. Paste the output in to a model, and refactor as required.
## create_base_models ([source](macros/create_base_models.sql))
This macro generates a series of terminal commands (appended with the `&&` to allow for subsequent execution) that execute the [base_model_creation](#base_model_creation-source) bash script. This bash script will write the output of the [generate_base_model](#generate_base_model-source) macro into a new model file in your local dbt project.
> **Note**: This macro is not compatible with the dbt Cloud IDE.
### Arguments:
- `source_name` (required): The source you wish to generate base model SQL for.
- `tables` (required): A list of all tables you want to generate the base models for.
### Usage:
1. Create a source for the table you wish to create a base model on top of.
2. Copy the macro into a statement tab into your local IDE, and run your code
```sql
dbt run-operation codegen.create_base_models --args '{source_name: my-source, tables: ["this-table","that-table"]}'
```
## base_model_creation ([source](bash_scripts/base_model_creation.sh))
This bash script when executed from your local IDE will create model files in your dbt project instance that contain the outputs of the [generate_base_model](macros/generate_base_model.sql) macro.
> **Note**: This macro is not compatible with the dbt Cloud IDE.
### Arguments:
- `source_name` (required): The source you wish to generate base model SQL for.
- `table_name` (required): A single table name for which you want to generate the base model.
### Usage:
1. Create a source for the table you wish to create a base model on top of.
2. Copy the macro into a statement tab into your local IDE, and run your code
```bash
source dbt_packages/codegen/bash_scripts/base_model_creation.sh "source_name" "table_name"
```
## generate_model_yaml ([source](macros/generate_model_yaml.sql))
This macro generates the YAML for a list of model(s), which you can then paste into a
schema.yml file.
### Arguments:
- `model_names` (required): The model(s) you wish to generate YAML for.
- `upstream_descriptions` (optional, default=False): Whether you want to include descriptions for identical column names from upstream models and sources.
- `include_data_types` (optional, default=True): Whether you want to add data types to your model column definitions.
### Usage:
1. Create a model.
2. Copy the macro into a statement tab in the dbt Cloud IDE, or into an analysis file, and compile your code
```
{{ codegen.generate_model_yaml(
model_names=['customers']
) }}
```
You can use the helper function codegen.get_models and specify a directory and/or prefix to get a list of all matching models, to be passed into model_names list.
```
{% set models_to_generate = codegen.get_models(directory='marts', prefix='fct_') %}
{{ codegen.generate_model_yaml(
model_names = models_to_generate
) }}
```
Alternatively, call the macro as an [operation](https://docs.getdbt.com/docs/using-operations):
```
$ dbt run-operation generate_model_yaml --args '{"model_names": ["customers"]}'
```
3. The YAML for a base model(s) will be logged to the command line
```
version: 2
models:
- name: customers
description: ""
columns:
- name: customer_id
data_type: integer
description: ""
- name: customer_name
data_type: text
description: ""
```
4. Paste the output in to a schema.yml file, and refactor as required.
## generate_model_import_ctes ([source](macros/generate_model_import_ctes.sql))
This macro generates the SQL for a given model with all references pulled up into import CTEs, which you can then paste back into the model.
### Arguments:
- `model_name` (required): The model you wish to generate SQL with import CTEs for.
- `leading_commas` (optional, default=False): Whether you want your commas to be leading (vs trailing).
### Usage:
1. Create a model with your original SQL query
2. Copy the macro into a statement tab in the dbt Cloud IDE, or into an analysis file, and compile your code
```
{{ codegen.generate_model_import_ctes(
model_name = 'my_dbt_model'
) }}
```
Alternatively, call the macro as an [operation](https://docs.getdbt.com/docs/using-operations):
```
$ dbt run-operation generate_model_import_ctes --args '{"model_name": "my_dbt_model"}'
```
3. The new SQL - with all references pulled up into import CTEs - will be logged to the command line
```
with customers as (
select * from {{ ref('stg_customers') }}
),
orders as (
select * from {{ ref('stg_orders') }}
),
payments as (
select * from {{ ref('stg_payments') }}
),
customer_orders as (
select
customer_id,
min(order_date) as first_order,
max(order_date) as most_recent_order,
count(order_id) as number_of_orders
from orders
group by customer_id
),
customer_payments as (
select
orders.customer_id,
sum(amount) as total_amount
from payments
left join orders on
payments.order_id = orders.order_id
group by orders.customer_id
),
final as (
select
customers.customer_id,
customers.first_name,
customers.last_name,
customer_orders.first_order,
customer_orders.most_recent_order,
customer_orders.number_of_orders,
customer_payments.total_amount as customer_lifetime_value
from customers
left join customer_orders
on customers.customer_id = customer_orders.customer_id
left join customer_payments
on customers.customer_id = customer_payments.customer_id
)
select * from final
```
4. Replace the contents of the model's current SQL file with the compiled or logged code
## generate_unit_test_template ([source](macros/generate_unit_test_template.sql))
This macro generates the unit testing YAML for a given model with all references included as `given` inputs (along with their columns), plus the columns within the expected output.
### Arguments:
- `model_name` (required): The model you wish to generate unit testing YAML for.
- `inline_columns` (optional, default=False): Whether you want all columns on the same line.
### Usage:
1. Create a model with your original SQL query
2. Call the macro as an [operation](https://docs.getdbt.com/docs/using-operations):
```
$ dbt run-operation generate_unit_test_template --args '{"model_name": "order_items", "inline_columns": true}'
```
3. The new YAML - with all given inputs included - will be logged to the command line
```yaml
unit_tests:
- name: unit_test_order_items
model: order_items
given:
- input: ref("stg_order_items")
rows:
- col_a:
col_b:
expect:
rows:
- id:
```
4. Create a new YAML file with the compiled or logged code.
5. Add column values for the given inputs and expected output.
## Contributing
To contirbute code to this package, please follow the steps outlined in the `integration_tests` directory's [README](https://github.com/dbt-labs/dbt-codegen/blob/main/integration_tests/README.md) file.
================================================
FILE: RELEASE.md
================================================
# dbt-codegen releases
## When do we release?
There's a few scenarios that might prompt a release:
| Scenario | Release type |
|--------------------------------------------|--------------|
| Breaking changes to existing macros | minor |
| New functionality | minor |
| Fixes to existing macros | patch |
## Release process
1. Begin a new release by clicking [here](https://github.com/dbt-labs/dbt-codegen/releases/new)
1. Click "Choose a tag", then paste your version number (with no "v" in the name), then click "Create new tag: x.y.z. on publish"
- The “Release title” will be identical to the tag name
1. Click the "Generate release notes" button
1. Copy and paste the generated release notes into `CHANGELOG.md`, commit, and merge into the `main` branch
1. Click the "Publish release" button
- This will automatically create an "Assets" section containing:
- Source code (zip)
- Source code (tar.gz)
================================================
FILE: bash_scripts/base_model_creation.sh
================================================
#!/bin/bash
echo "" > models/stg_$1__$2.sql
dbt --quiet run-operation codegen.generate_base_model --args '{"source_name": "'$1'", "table_name": "'$2'"}' | tail -n +3 >> models/stg_$1__$2.sql
================================================
FILE: dbt_project.yml
================================================
name: "codegen"
version: "0.5.0"
require-dbt-version: [">=1.1.0", "<3.0.0"]
config-version: 2
target-path: "target"
clean-targets: ["target", "dbt_packages"]
macro-paths: ["macros"]
log-path: "logs"
================================================
FILE: docker-compose.yml
================================================
version: "3.7"
services:
postgres:
image: cimg/postgres:17.0
environment:
- POSTGRES_USER=root
ports:
- "5432:5432"
================================================
FILE: integration_tests/README.md
================================================
## Table of Contents
1. [Overview](#overview)
1. [Prerequisites](#prerequisites)
2. [Introduction](#introduction)
2. [Setup](#setup)
1. [Configure credentials](#configure-credentials)
2. [Setup Postgres or other database targets](#setup-postgres-or-other-database-targets)
3. [Set up virtual environment](#set-up-virtual-environment)
4. [Install dependencies](#install-dependencies)
3. [Write or modify an integration test](#write-or-modify-an-integration-test)
1. [Run the integration tests](#run-the-integration-tests)
2. [Creating a new integration test](#creating-a-new-integration-test)
4. [Implement the functionality](#implement-the-functionality)
5. [Commit your changes and open a pull request](#commit-your-changes-and-open-a-pull-request)
## Overview
### Prerequisites
- [python3](https://www.python.org/)
- [make](<https://en.wikipedia.org/wiki/Make_(software)>) (Optional, but recommended for better development experience)[^1]
- [Docker](https://www.docker.com/) (Optional, but recommended for using Postgres as your target database easily)[^2]
### Introduction
Packages in dbt are actually dbt projects themselves, you write SQL and Jinja, sometimes in macros, to add new functionality or models to another dbt project. As SQL and Jinja rely on input data, it's essential to have a functioning project to be able to test that the code works as expected. Constantly running the code, loading data, running bits and pieces, and hoping for the best is not a good development flow though, nor is it a reliable way to ensure that everything works. This is why our dbt packages have integration tests. These tests run all of the data loading, model building, and tests that are defined in the package inside testing environments, and check that the results are as expected.
If you add or modify functionality in any codegen macros, there should be corresponding changes to the integration tests. This README will walk you through this process. Let's outline the basic steps first:
1. Set up your environment (credentials, virtual environment, dependencies, test database(s))
2. Write or modify an integration test (you should expect this to fail as you haven't implemented the functionality yet!)
3. Implement the functionality in the new or modified macro, and run the tests to get them to pass.
4. Commit your changes and open a pull request.
## Setup
### Configure credentials
You'll need to set environment variables with the credentials to access your target database. If you're using the recommended local development path of Postgres in Docker, these values are already filled in as they are generic. For the cloud warehouses listed, you'll need real credentials. You probably want to ensure you're building into a testing schema as well to keep the output of this codegen separate from any production data. We run against all the warehouses listed in the CI (implmented via CircleCI) when you open a PR, so feel free to test against Postgres while developing, and we'll ensure the code works against all the other targets.
You can set these env vars in a couple ways:
- **Temporary**: Set these environment variables in your shell before running the tests. This is the easiest way to get started, but you'll have to set them every time you open a new terminal.
- **Reusable**: If you anticipate developing for multiple sessions, set these environment variables in your shell profile (like `~/.bashrc` or `~/.zshrc`). This way, you won't have to set them every time you open a new terminal.
The environment variables you'll need to set for each adapter can be found in [integration_tests/.env/](integration_tests/.env/).
### Setup Postgres or other database targets
As mentioned, you'll need a target database to run the integration tests and develop against. You can use a cloud warehouse, but the easiest and free way to work is to use Postgres locally. We include a `docker-compose.yml` file that will spin up a Postgres container for you to make this easy.
To run the Postgres container, just run:
```shell
make setup-db
```
Or, alternatively:
```shell
docker-compose up --detach postgres
```
> [!NOTE]
> `make` is a venerable build tool that is included in most Unix-like operating systems. It's not strictly necessary to use `make` to develop on this project, but there are several `make` commands that wrap more complex commands and make development easier. If you don't have `make` installed or don't want to use it, you can just run the commands in the `Makefile` directly. All the examples will show both options.
### Set up virtual environment
We strongly recommend using virtual environments when developing code in `dbt-codegen`. We recommend creating this virtual environment in the root of the `dbt-codegen` repository. To create a new virtual environment, run:
```shell
python3 -m venv .venv
source .venv/bin/activate
```
This will create and activate a new Python virtual environment.
### Install dependencies
First make sure that you set up your virtual environment as described above. Also ensure you have the latest version of pip and setuptools installed:
```
python3 -m pip install --upgrade pip setuptools
```
Next, install `dbt-core` (and its dependencies) with:
```shell
make dev target=[postgres|redshift|...]
# or
python3 -m pip install dbt-core dbt-[postgres|redshift|...]
```
Or more specific:
```shell
make dev target=postgres
# or
python3 -m pip install dbt-core dbt-postgres
```
Make sure to reload your virtual environment after installing the dependencies:
```shell
source .venv/bin/activate
```
## Write or modify an integration test
Run all the tests _before_ you start developing to make sure everything is working as expected before you start making changes. Nothing is worse than spending a ton of time troubleshooting a failing test, only to realize it was failing before you touched anything. This will also ensure that you have the correct environment variables set up and that your database is running.
### Run the Circle CI integration tests
To run all the integration tests on your local machine like they will get run in CI:
```shell
make test target=[postgres|redshift|...]
# or
./run_test.sh [postgres|redshift|...]
```
Or more specific:
```shell
make test target=postgres
# or
./run_test.sh postgres
```
### Run the tox Supported Tests
To run all the integration tests on your local machine like they will get run in the CI (using GitHub workflows with tox):
```shell
make test_tox target=postgres
```
### Creating a new integration test
Adding integration tests for new functionality typically involves making one or more of the following:
- a new seed file of fixture data
- a new model file to test against
- a new test to assert anticipated behaviour
Once you've added and/or edited the necessary files, assuming you are in the sub-project in the `integration_tests` folder, you should be able to run and test your new additions specifically by running:
```shell
dbt deps --target {your_target}
dbt build --target {your_target} --select +{your_selection_criteria}
```
The `dbt build` command will handle seeding, running, and testing the selection in a single command. The `+` operator in the `--select` flag indicates we also want to build everything that this selection depends on.
Or simply `make dev target={your_target}` and then `make test target={your_target}` if you're okay with running the entire project and all tests.
Remember, typically you'll want to create a failing test _first_, then implement the functionality to make it pass. This is called "test-driven development" (TDD) and it's a great way to ensure that your code really does what you expect it to. For example, let's imagine you wrote a test expecting it to fail, but it passed before you even implemented your logic! That would mean the test is not actually testing what you want, and you'd need to re-evaluate your assumptions. That's something you want to catch early in the development process, and what TDD is all about. So, expect this run of tests after you add your new logic to fail.
## Implement the functionality
Okay finally, this is the fun part! You can now implement the functionality in the macro you're working on.The development flow should be something like:
1. You've got a failing test, so you know what you need to implement.
2. Implement some logic in the macro you're working on.
3. Run the relevant tests to see if they pass.
4. Repeat until the tests pass.
5. Run the full test suite to ensure you didn't break anything else by accident.
## Commit your changes and open a pull request
Once your tests are passing and you're happy with the code, you'll want to commit it and open a new PR on GitHub. Don't forget to run the full test suite against your target database before you open a PR to make sure you didn't accidentally break any existing functionality. When you open a PR, CircleCI will run the same test suite against all the database targets. If they're passing, we'll triage and review the code as soon as we can! Thank you for contributing to dbt-codegen!
[^1]: If you're on a Mac, `make` is probably best installed with the XCode Command Line Tools, or you can install `make` via Homebrew with `brew install cmake`. If you're on Windows, you can either use the Windows Subsystem for Linux (WSL) or use `scoop` or `chocolatey` to install `make`. If you're on Linux, you probably already have `make` installed.
[^2]: Specific instructions on installing and getting started with Docker for your OS can be found [here](https://docs.docker.com/get-docker/).
================================================
FILE: integration_tests/dbt_project.yml
================================================
name: "codegen_integration_tests"
version: "1.0"
config-version: 2
profile: "integration_tests"
model-paths: ["models"]
analysis-paths: ["analyses"]
test-paths: ["tests"]
seed-paths: ["seeds"]
macro-paths: ["macros"]
target-path: "target"
clean-targets:
- "target"
- "dbt_packages"
flags:
send_anonymous_usage_stats: False
use_colors: True
seeds:
+schema: raw_data
+quote_columns: false
vars:
my_table_reference: table_c
models:
+bind: false
================================================
FILE: integration_tests/macros/assert_equal.sql
================================================
{% macro assert_equal(actual_object, expected_object) %}
{% if not execute %}
{# pass #}
{% elif actual_object != expected_object %}
{% set msg %}
Expected did not match actual
-----------
Actual:
-----------
--->{{ actual_object }}<---
-----------
Expected:
-----------
--->{{ expected_object }}<---
{% endset %}
{{ log(msg, info=True) }}
select 'fail'
{% else %}
select 'ok' limit 0
{% endif %}
{% endmacro %}
================================================
FILE: integration_tests/macros/integer_type_value.sql
================================================
{%- macro integer_type_value() -%}
{%- if target.type == "snowflake" -%}
number
{%- elif target.type == "bigquery" -%}
int64
{%- else -%}
integer
{%- endif -%}
{%- endmacro -%}
================================================
FILE: integration_tests/macros/operations/create_source_table.sql
================================================
{% macro create_source_table() %}
{% set target_schema=api.Relation.create(
database=target.database,
schema=target.schema ~ "__data_source_schema"
) %}
{% do adapter.create_schema(target_schema) %}
{% set drop_table_sql %}
drop table if exists {{ target_schema }}.codegen_integration_tests__data_source_table {% if target.type == "redshift" %}cascade{% endif %}
{% endset %}
{{ run_query(drop_table_sql) }}
{% set create_table_sql %}
create table {{ target_schema }}.codegen_integration_tests__data_source_table as (
select
1 as my_integer_col,
true as my_bool_col
)
{% endset %}
{{ run_query(create_table_sql) }}
{% set drop_table_sql_case_sensitive %}
drop table if exists {{ target_schema }}.codegen_integration_tests__data_source_table_case_sensitive {% if target.type == "redshift" %}cascade{% endif %}
{% endset %}
{{ run_query(drop_table_sql_case_sensitive) }}
{% set create_table_sql_case_sensitive %}
create table {{ target_schema }}.codegen_integration_tests__data_source_table_case_sensitive as (
select
1 as {{ adapter.quote("My_Integer_Col") }},
true as {{ adapter.quote("My_Bool_Col") }}
)
{% endset %}
{{ run_query(create_table_sql_case_sensitive) }}
{% endmacro %}
================================================
FILE: integration_tests/macros/text_type_value.sql
================================================
{%- macro text_type_value() -%}
{%- if target.type == "redshift"-%}
character varying
{%- elif target.type == "snowflake" -%}
varchar
{%- elif target.type == "bigquery" -%}
string
{%- else -%}
text
{%- endif -%}
{%- endmacro -%}
================================================
FILE: integration_tests/models/child_model.sql
================================================
select
*
from {{ ref('model_data_a') }}
================================================
FILE: integration_tests/models/model_data_a.sql
================================================
select
*
from {{ ref('data__a_relation') }}
================================================
FILE: integration_tests/models/model_from_source.sql
================================================
select
*
from {{ source('codegen_integration_tests__data_source_schema', 'codegen_integration_tests__data_source_table') }}
================================================
FILE: integration_tests/models/model_incremental.sql
================================================
{{ config(
materialized='incremental'
) }}
select 1 as id
================================================
FILE: integration_tests/models/model_repeated.sql
================================================
{% if target.type == "bigquery" %}
{#--- This exists to test the BigQuery-specific behavior requested in #190 -#}
select
[1, 2] AS repeated_int,
[
STRUCT(1 as nested_int_field, [STRUCT("a" as string_field)] as nested_repeated_struct),
STRUCT(2 AS nested_int_field, [STRUCT("a" as string_field)] as nested_repeated_struct)
] as repeated_struct
{% else %}
select 1 as int_field
{% endif %}
================================================
FILE: integration_tests/models/model_struct.sql
================================================
{% if target.type == "bigquery" %}
{#--- This exists to test the BigQuery-specific behavior requested in #27 -#}
select
STRUCT(
source,
medium,
source_medium
) as analytics,
col_x
from {{ ref('data__campaign_analytics') }}
{% else %}
{#--- This enables mimicking the BigQuery behavior for other adapters -#}
select
analytics,
source,
medium,
source_medium,
col_x
from {{ ref('data__campaign_analytics') }}
{% endif %}
================================================
FILE: integration_tests/models/model_without_any_ctes.sql
================================================
select *, 2 as col2
from {{ ref('model_without_import_ctes') }} as m
left join (select 2 as col_a from {{ ref('data__a_relation') }}) as a on a.col_a = m.id
where id = 1
================================================
FILE: integration_tests/models/model_without_import_ctes.sql
================================================
/*
This is my model!
*/
{{ config(
materialized='table',
) }}
-- I love this cte
with my_first_cte as (
select
a.col_a,
b.col_b
from {{ ref('data__a_relation') }} as a
left join {{ ref("data__b_relation") }} as b
on a.col_a = b.col_a
left join {{ ref('data__a_relation') }} as aa
on a.col_a = aa.col_a
),
my_second_cte as (
select
1 as id
from {{ target.schema }}__data_source_schema.codegen_integration_tests__data_source_table
union all
select
2 as id
from {{ source('codegen_integration_tests__data_source_schema', 'codegen_integration_tests__data_source_table') }}
-- union all
-- select
-- 3 as id
-- from development.codegen_integration_tests__data_source_schema.codegen_integration_tests__data_source_table
-- union all
-- select
-- 4 as id
-- from {{ var("my_table_reference") }}
-- union all
-- select
-- 5 as id
-- from {{ var("my_other_table_reference", "table_d") }}
)
-- my_third_cte as (
-- select
-- a.col_a,
-- b.col_b
-- from `raw_relation_1` as a
-- left join "raw_relation_2" as b
-- on a.col_a = b.col_b
-- left join [raw_relation_3] as aa
-- on a.col_a = aa.col_b
-- left join 'raw_relation_4' as ab
-- on a.col_a = ab.col_b
-- left join 'my_schema'.'raw_relation_5' as ac
-- on a.col_a = ac.col_b
-- )
select * from my_second_cte
================================================
FILE: integration_tests/models/schema.yml
================================================
version: 2
models:
- name: model_data_a
columns:
- name: col_a
description: 'description column "a"'
================================================
FILE: integration_tests/models/source.yml
================================================
version: 2
sources:
- name: codegen_integration_tests__data_source_schema
schema: "{{ target.schema ~ '__data_source_schema' }}"
tables:
- name: codegen_integration_tests__data_source_table
columns:
- name: my_integer_col
description: My Integer Column
- name: my_bool_col
description: My Boolean Column
- name: codegen_integration_tests__data_source_table_case_sensitive
================================================
FILE: integration_tests/package-lock.yml
================================================
packages:
- local: ../
- package: dbt-labs/dbt_utils
version: 1.1.1
sha1_hash: de2deba3d66ce03d8c02949013650cc9b94f6030
================================================
FILE: integration_tests/packages.yml
================================================
packages:
- local: ../
================================================
FILE: integration_tests/profiles.yml
================================================
# HEY! This file is used in the dbt-codegen integrations tests with GitHub CI.
# You should __NEVER__ check credentials into version control. That's why we use environment variables everywhere.
# Thanks for reading :)
integration_tests:
target: postgres
outputs:
postgres:
type: "postgres"
host: "{{ env_var('POSTGRES_HOST') }}"
user: "{{ env_var('POSTGRES_USER') }}"
pass: "{{ env_var('DBT_ENV_SECRET_POSTGRES_PASS') }}"
port: "{{ env_var('POSTGRES_PORT') | as_number }}"
dbname: "{{ env_var('POSTGRES_DATABASE') }}"
schema: "{{ env_var('POSTGRES_SCHEMA') }}"
threads: 5
redshift:
type: "redshift"
host: "{{ env_var('REDSHIFT_HOST') }}"
user: "{{ env_var('REDSHIFT_USER') }}"
pass: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_PASS') }}"
dbname: "{{ env_var('REDSHIFT_DATABASE') }}"
port: "{{ env_var('REDSHIFT_PORT') | as_number }}"
schema: "{{ env_var('REDSHIFT_SCHEMA') }}"
threads: 1
bigquery:
type: bigquery
method: service-account-json
keyfile_json:
"{{ env_var('BIGQUERY_KEYFILE_JSON') | as_native}}"
project: "{{ env_var('BIGQUERY_PROJECT') }}"
dataset: "{{ env_var('BIGQUERY_SCHEMA') }}"
threads: 1
snowflake:
type: "snowflake"
account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}"
user: "{{ env_var('SNOWFLAKE_USER') }}"
password: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_PASS') }}"
role: "{{ env_var('SNOWFLAKE_ROLE') }}"
database: "{{ env_var('SNOWFLAKE_DATABASE') }}"
warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') }}"
schema: "{{ env_var('SNOWFLAKE_SCHEMA') }}"
threads: 1
================================================
FILE: integration_tests/seeds/data__a_relation.csv
================================================
col_a,col_b
1,a
2,b
================================================
FILE: integration_tests/seeds/data__b_relation.csv
================================================
col_a,col_b
3,c
4,d
================================================
FILE: integration_tests/seeds/data__campaign_analytics.csv
================================================
source,medium,source_medium,analytics,col_x
source_1,medium_a,1a,,x
source_2,medium_b,2b,,x
source_3,medium_c,3c,,x
================================================
FILE: integration_tests/tests/test_generate_base_models.sql
================================================
{% set actual_base_model = codegen.generate_base_model(
source_name='codegen_integration_tests__data_source_schema',
table_name='codegen_integration_tests__data_source_table'
)
%}
{% set expected_base_model %}
with source as (
select * from {%raw%}{{ source('codegen_integration_tests__data_source_schema', 'codegen_integration_tests__data_source_table') }}{%endraw%}
),
renamed as (
select
my_integer_col,
my_bool_col
from source
)
select * from renamed
{% endset %}
{{ assert_equal (actual_base_model | trim, expected_base_model | trim) }}
================================================
FILE: integration_tests/tests/test_generate_base_models_all_args.sql
================================================
{% set actual_base_model = codegen.generate_base_model(
source_name='codegen_integration_tests__data_source_schema',
table_name='codegen_integration_tests__data_source_table_case_sensitive',
leading_commas=True,
case_sensitive_cols=True,
materialized='table'
)
%}
{% set expected_base_model %}
{{ "{{ config(materialized='table') }}" }}
with source as (
select * from {%raw%}{{ source('codegen_integration_tests__data_source_schema', 'codegen_integration_tests__data_source_table_case_sensitive') }}{%endraw%}
),
renamed as (
select
{{ adapter.quote("My_Integer_Col") }}
, {{ adapter.quote("My_Bool_Col") }}
from source
)
select * from renamed
{% endset %}
{{ assert_equal (actual_base_model | trim, expected_base_model | trim) }}
================================================
FILE: integration_tests/tests/test_generate_base_models_case_sensitive.sql
================================================
{% set actual_base_model = codegen.generate_base_model(
source_name='codegen_integration_tests__data_source_schema',
table_name='codegen_integration_tests__data_source_table_case_sensitive',
case_sensitive_cols=True
)
%}
{% set expected_base_model %}
with source as (
select * from {%raw%}{{ source('codegen_integration_tests__data_source_schema', 'codegen_integration_tests__data_source_table_case_sensitive') }}{%endraw%}
),
renamed as (
select
{{ adapter.quote("My_Integer_Col") }},
{{ adapter.quote("My_Bool_Col") }}
from source
)
select * from renamed
{% endset %}
{{ assert_equal (actual_base_model | trim, expected_base_model | trim) }}
================================================
FILE: integration_tests/tests/test_generate_base_models_leading.sql
================================================
{% set actual_base_model = codegen.generate_base_model(
source_name='codegen_integration_tests__data_source_schema',
table_name='codegen_integration_tests__data_source_table',
leading_commas=True
)
%}
{% set expected_base_model %}
with source as (
select * from {%raw%}{{ source('codegen_integration_tests__data_source_schema', 'codegen_integration_tests__data_source_table') }}{%endraw%}
),
renamed as (
select
my_integer_col
, my_bool_col
from source
)
select * from renamed
{% endset %}
{{ assert_equal (actual_base_model | trim, expected_base_model | trim) }}
================================================
FILE: integration_tests/tests/test_generate_model_import_ctes.sql
================================================
{% set actual_model_with_import_ctes = codegen.generate_model_import_ctes(
model_name = 'model_without_import_ctes'
)
%}
{% set expected_model_with_import_ctes %}
/*
This is my model!
*/
{% raw %}{{ config(
materialized='table',
) }}{% endraw %}
with data__a_relation as (
select * from {% raw %}{{ ref('data__a_relation') }}{% endraw %}
),
data__b_relation as (
select * from {% raw %}{{ ref("data__b_relation") }}{% endraw %}
),
development_codegen_integration_tests__data_source_schema_codegen_integration_tests__data_source_table as (
select * from development.codegen_integration_tests__data_source_schema.codegen_integration_tests__data_source_table
-- CAUTION: It's best practice to use the ref or source function instead of a direct reference
),
my_other_table_reference as (
select * from {% raw %}{{ var("my_other_table_reference", "table_d") }}{% endraw %}
-- CAUTION: It's best practice to use the ref or source function instead of a var
),
my_schema_raw_relation_5 as (
select * from 'my_schema'.'raw_relation_5'
-- CAUTION: It's best practice to use the ref or source function instead of a direct reference
),
my_table_reference as (
select * from {% raw %}{{ var("my_table_reference") }}{% endraw %}
-- CAUTION: It's best practice to use the ref or source function instead of a var
),
raw_relation_1 as (
select * from `raw_relation_1`
-- CAUTION: It's best practice to use the ref or source function instead of a direct reference
),
raw_relation_2 as (
select * from "raw_relation_2"
-- CAUTION: It's best practice to use the ref or source function instead of a direct reference
),
raw_relation_3 as (
select * from [raw_relation_3]
-- CAUTION: It's best practice to use the ref or source function instead of a direct reference
),
raw_relation_4 as (
select * from 'raw_relation_4'
-- CAUTION: It's best practice to use the ref or source function instead of a direct reference
),
source_codegen_integration_tests__data_source_table as (
select * from {% raw %}{{ source('codegen_integration_tests__data_source_schema', 'codegen_integration_tests__data_source_table') }}{% endraw %}
-- CAUTION: It's best practice to create staging layer for raw sources
),
-- I love this cte
my_first_cte as (
select
a.col_a,
b.col_b
from data__a_relation as a
left join data__b_relation as b
on a.col_a = b.col_a
left join data__a_relation as aa
on a.col_a = aa.col_a
),
my_second_cte as (
select
1 as id
from {% raw %}{{ target.schema }}{% endraw %}__data_source_schema.codegen_integration_tests__data_source_table
union all
select
2 as id
from source_codegen_integration_tests__data_source_table
-- union all
-- select
-- 3 as id
-- from development_codegen_integration_tests__data_source_schema_codegen_integration_tests__data_source_table
-- union all
-- select
-- 4 as id
-- from my_table_reference
-- union all
-- select
-- 5 as id
-- from my_other_table_reference
)
-- my_third_cte as (
-- select
-- a.col_a,
-- b.col_b
-- from raw_relation_1 as a
-- left join raw_relation_2 as b
-- on a.col_a = b.col_b
-- left join raw_relation_3 as aa
-- on a.col_a = aa.col_b
-- left join raw_relation_4 as ab
-- on a.col_a = ab.col_b
-- left join my_schema_raw_relation_5 as ac
-- on a.col_a = ac.col_b
-- )
select * from my_second_cte
{% endset %}
{{ assert_equal (actual_model_with_import_ctes | trim, expected_model_with_import_ctes | trim) }}
================================================
FILE: integration_tests/tests/test_generate_model_import_ctes_leading.sql
================================================
{% set actual_model_with_import_ctes = codegen.generate_model_import_ctes(
model_name = 'model_without_import_ctes',
leading_commas = true
)
%}
{% set expected_model_with_import_ctes %}
/*
This is my model!
*/
{% raw %}{{ config(
materialized='table',
) }}{% endraw %}
with data__a_relation as (
select * from {% raw %}{{ ref('data__a_relation') }}{% endraw %}
)
,data__b_relation as (
select * from {% raw %}{{ ref("data__b_relation") }}{% endraw %}
)
,development_codegen_integration_tests__data_source_schema_codegen_integration_tests__data_source_table as (
select * from development.codegen_integration_tests__data_source_schema.codegen_integration_tests__data_source_table
-- CAUTION: It's best practice to use the ref or source function instead of a direct reference
)
,my_other_table_reference as (
select * from {% raw %}{{ var("my_other_table_reference", "table_d") }}{% endraw %}
-- CAUTION: It's best practice to use the ref or source function instead of a var
)
,my_schema_raw_relation_5 as (
select * from 'my_schema'.'raw_relation_5'
-- CAUTION: It's best practice to use the ref or source function instead of a direct reference
)
,my_table_reference as (
select * from {% raw %}{{ var("my_table_reference") }}{% endraw %}
-- CAUTION: It's best practice to use the ref or source function instead of a var
)
,raw_relation_1 as (
select * from `raw_relation_1`
-- CAUTION: It's best practice to use the ref or source function instead of a direct reference
)
,raw_relation_2 as (
select * from "raw_relation_2"
-- CAUTION: It's best practice to use the ref or source function instead of a direct reference
)
,raw_relation_3 as (
select * from [raw_relation_3]
-- CAUTION: It's best practice to use the ref or source function instead of a direct reference
)
,raw_relation_4 as (
select * from 'raw_relation_4'
-- CAUTION: It's best practice to use the ref or source function instead of a direct reference
)
,source_codegen_integration_tests__data_source_table as (
select * from {% raw %}{{ source('codegen_integration_tests__data_source_schema', 'codegen_integration_tests__data_source_table') }}{% endraw %}
-- CAUTION: It's best practice to create staging layer for raw sources
)
-- I love this cte
,my_first_cte as (
select
a.col_a,
b.col_b
from data__a_relation as a
left join data__b_relation as b
on a.col_a = b.col_a
left join data__a_relation as aa
on a.col_a = aa.col_a
),
my_second_cte as (
select
1 as id
from {% raw %}{{ target.schema }}{% endraw %}__data_source_schema.codegen_integration_tests__data_source_table
union all
select
2 as id
from source_codegen_integration_tests__data_source_table
-- union all
-- select
-- 3 as id
-- from development_codegen_integration_tests__data_source_schema_codegen_integration_tests__data_source_table
-- union all
-- select
-- 4 as id
-- from my_table_reference
-- union all
-- select
-- 5 as id
-- from my_other_table_reference
)
-- my_third_cte as (
-- select
-- a.col_a,
-- b.col_b
-- from raw_relation_1 as a
-- left join raw_relation_2 as b
-- on a.col_a = b.col_b
-- left join raw_relation_3 as aa
-- on a.col_a = aa.col_b
-- left join raw_relation_4 as ab
-- on a.col_a = ab.col_b
-- left join my_schema_raw_relation_5 as ac
-- on a.col_a = ac.col_b
-- )
select * from my_second_cte
{% endset %}
{{ assert_equal (actual_model_with_import_ctes | trim, expected_model_with_import_ctes | trim) }}
================================================
FILE: integration_tests/tests/test_generate_model_import_ctes_no_ctes.sql
================================================
{% set actual_model_with_import_ctes = codegen.generate_model_import_ctes(
model_name = 'model_without_any_ctes'
)
%}
{% set expected_model_with_import_ctes %}
with data__a_relation as (
select * from {% raw %}{{ ref('data__a_relation') }}{% endraw %}
),
model_without_import_ctes as (
select * from {% raw %}{{ ref('model_without_import_ctes') }}{% endraw %}
)
select *, 2 as col2
from model_without_import_ctes as m
left join (select 2 as col_a from data__a_relation) as a on a.col_a = m.id
where id = 1
{% endset %}
{{ assert_equal (actual_model_with_import_ctes | trim, expected_model_with_import_ctes | trim) }}
================================================
FILE: integration_tests/tests/test_generate_model_repeated_yaml.sql
================================================
{% set raw_schema = generate_schema_name('raw_data') %}
{% set actual_source_yaml = codegen.generate_model_yaml(
model_names=['model_repeated']
)
%}
{% if target.type == "bigquery" %}
{% set expected_source_yaml %}
version: 2
models:
- name: model_repeated
description: ""
columns:
- name: repeated_int
data_type: array<int64>
description: ""
- name: repeated_struct
data_type: array
description: ""
- name: repeated_struct.nested_int_field
data_type: int64
description: ""
- name: repeated_struct.nested_repeated_struct
data_type: array
description: ""
- name: repeated_struct.nested_repeated_struct.string_field
data_type: string
description: ""
{% endset %}
{% else %}
{% set expected_source_yaml %}
version: 2
models:
- name: model_repeated
description: ""
columns:
- name: int_field
data_type: {{ integer_type_value() }}
description: ""
{% endset %}
{% endif %}
{{ assert_equal (actual_source_yaml | trim, expected_source_yaml | trim) }}
================================================
FILE: integration_tests/tests/test_generate_model_struct_yaml.sql
================================================
{% set raw_schema = generate_schema_name('raw_data') %}
-- test all args
{% set actual_source_yaml = codegen.generate_source(
database_name=target.database,
schema_name='codegen_integration_tests__data_source_schema',
table_names=['codegen_integration_tests__data_source_table_nested_array'],
generate_columns=True,
include_descriptions=True
) %}
{% set actual_source_yaml = codegen.generate_model_yaml(
model_names=['model_struct'],
include_data_types=False
)
%}
{% if target.type == "bigquery" %}
{% set expected_source_yaml %}
version: 2
models:
- name: model_struct
description: ""
columns:
- name: analytics
description: ""
- name: analytics.source
description: ""
- name: analytics.medium
description: ""
- name: analytics.source_medium
description: ""
- name: col_x
description: ""
{% endset %}
{% else %}
{% set expected_source_yaml %}
version: 2
models:
- name: model_struct
description: ""
columns:
- name: analytics
description: ""
- name: source
description: ""
- name: medium
description: ""
- name: source_medium
description: ""
- name: col_x
description: ""
{% endset %}
{% endif %}
{{ assert_equal (actual_source_yaml | trim, expected_source_yaml | trim) }}
================================================
FILE: integration_tests/tests/test_generate_model_yaml.sql
================================================
{% set actual_model_yaml = codegen.generate_model_yaml(
model_names=['data__a_relation']
)
%}
{% set expected_model_yaml %}
version: 2
models:
- name: data__a_relation
description: ""
columns:
- name: col_a
data_type: {{ integer_type_value() }}
description: ""
- name: col_b
data_type: {{ text_type_value() }}
description: ""
{% endset %}
{{ assert_equal (actual_model_yaml | trim, expected_model_yaml | trim) }}
================================================
FILE: integration_tests/tests/test_generate_model_yaml_multiple_models.sql
================================================
{% set actual_model_yaml = codegen.generate_model_yaml(
model_names=['data__a_relation','data__b_relation'],
include_data_types=False
)
%}
{% set expected_model_yaml %}
version: 2
models:
- name: data__a_relation
description: ""
columns:
- name: col_a
description: ""
- name: col_b
description: ""
- name: data__b_relation
description: ""
columns:
- name: col_a
description: ""
- name: col_b
description: ""
{% endset %}
{{ assert_equal (actual_model_yaml | trim, expected_model_yaml | trim) }}
================================================
FILE: integration_tests/tests/test_generate_model_yaml_upstream_descriptions.sql
================================================
{% set actual_model_yaml = codegen.generate_model_yaml(
model_names=['child_model'],
upstream_descriptions=True,
include_data_types=False
)
%}
{% set expected_model_yaml %}
version: 2
models:
- name: child_model
description: ""
columns:
- name: col_a
description: "description column \"a\""
- name: col_b
description: ""
{% endset %}
{{ assert_equal (actual_model_yaml | trim, expected_model_yaml | trim) }}
================================================
FILE: integration_tests/tests/test_generate_model_yaml_upstream_source_descriptions.sql
================================================
{% set actual_model_yaml = codegen.generate_model_yaml(
model_names=['model_from_source'],
upstream_descriptions=True,
include_data_types=False
)
%}
{% set expected_model_yaml %}
version: 2
models:
- name: model_from_source
description: ""
columns:
- name: my_integer_col
description: "My Integer Column"
- name: my_bool_col
description: "My Boolean Column"
{% endset %}
{{ assert_equal (actual_model_yaml | trim, expected_model_yaml | trim) }}
================================================
FILE: integration_tests/tests/test_generate_source.sql
================================================
{% set raw_schema = generate_schema_name('raw_data') %}
-- test default args
{% set actual_source_yaml = codegen.generate_source(raw_schema) %}
{% set expected_source_yaml %}
version: 2
sources:
- name: {{ raw_schema | trim | lower }}
tables:
- name: data__a_relation
- name: data__b_relation
- name: data__campaign_analytics
{% endset %}
{{ assert_equal (actual_source_yaml | trim, expected_source_yaml | trim) }}
================================================
FILE: integration_tests/tests/test_generate_source_all_args.sql
================================================
{% set raw_schema = generate_schema_name('raw_data') %}
-- test all args
{% set actual_source_yaml = codegen.generate_source(
schema_name=raw_schema,
table_pattern='%',
exclude='',
database_name=target.database,
generate_columns=True,
include_descriptions=True,
include_data_types=True,
name=raw_schema,
table_names=None,
include_database=True,
include_schema=True,
case_sensitive_databases=False,
case_sensitive_schemas=False,
case_sensitive_tables=False,
case_sensitive_cols=False
) %}
{% set expected_source_yaml %}
version: 2
sources:
- name: {{ raw_schema | trim | lower }}
description: ""
database: {{ target.database | trim | lower }}
schema: {{ raw_schema | trim | lower }}
tables:
- name: data__a_relation
description: ""
columns:
- name: col_a
data_type: {{ integer_type_value() }}
description: ""
- name: col_b
data_type: {{ text_type_value() }}
description: ""
- name: data__b_relation
description: ""
columns:
- name: col_a
data_type: {{ integer_type_value() }}
description: ""
- name: col_b
data_type: {{ text_type_value() }}
description: ""
- name: data__campaign_analytics
description: ""
columns:
- name: source
data_type: {{ text_type_value() }}
description: ""
- name: medium
data_type: {{ text_type_value() }}
description: ""
- name: source_medium
data_type: {{ text_type_value() }}
description: ""
- name: analytics
data_type: {{ integer_type_value() }}
description: ""
- name: col_x
data_type: {{ text_type_value() }}
description: ""
{% endset %}
{{ assert_equal (actual_source_yaml | trim, expected_source_yaml | trim) }}
================================================
FILE: integration_tests/tests/test_generate_source_exclude.sql
================================================
{% set raw_schema = generate_schema_name('raw_data') %}
-- test default args
{% set actual_source_yaml = codegen.generate_source(raw_schema, table_pattern='data__%', exclude='data__a_%') %}
{% set expected_source_yaml %}
version: 2
sources:
- name: {{ raw_schema | trim | lower}}
tables:
- name: data__b_relation
- name: data__campaign_analytics
{% endset %}
{{ assert_equal (actual_source_yaml | trim, expected_source_yaml | trim) }}
================================================
FILE: integration_tests/tests/test_generate_source_include_database_property.sql
================================================
{% set raw_schema = generate_schema_name('raw_data') %}
{% set actual_source_yaml = codegen.generate_source(raw_schema, include_database=True) %}
{% set expected_source_yaml %}
version: 2
sources:
- name: {{ raw_schema | trim | lower }}
database: {{ target.database | trim | lower }}
tables:
- name: data__a_relation
- name: data__b_relation
- name: data__campaign_analytics
{% endset %}
{{ assert_equal (actual_source_yaml | trim, expected_source_yaml | trim) }}
================================================
FILE: integration_tests/tests/test_generate_source_include_schema_property.sql
================================================
{% set raw_schema = generate_schema_name('raw_data') %}
{% set actual_source_yaml = codegen.generate_source(raw_schema, include_schema=True) %}
{% set expected_source_yaml %}
version: 2
sources:
- name: {{ raw_schema | trim | lower }}
schema: {{ raw_schema | trim | lower }}
tables:
- name: data__a_relation
- name: data__b_relation
- name: data__campaign_analytics
{% endset %}
{{ assert_equal (actual_source_yaml | trim, expected_source_yaml | trim) }}
================================================
FILE: integration_tests/tests/test_generate_source_some_tables.sql
================================================
{% set raw_schema = generate_schema_name('raw_data') %}
-- test all args
{% set actual_source_yaml = codegen.generate_source(
schema_name=raw_schema,
database_name=target.database,
table_names=['data__a_relation'],
generate_columns=True,
include_descriptions=True,
include_data_types=False
) %}
{% set expected_source_yaml %}
version: 2
sources:
- name: {{ raw_schema | trim | lower }}
description: ""
tables:
- name: data__a_relation
description: ""
columns:
- name: col_a
description: ""
- name: col_b
description: ""
{% endset %}
{{ assert_equal (actual_source_yaml | trim, expected_source_yaml | trim) }}
================================================
FILE: integration_tests/tests/test_generate_source_table_descriptions.sql
================================================
{% set raw_schema = generate_schema_name('raw_data') %}
-- test default args
{% set actual_source_yaml = codegen.generate_source(raw_schema, include_descriptions=True) %}
{% set expected_source_yaml %}
version: 2
sources:
- name: {{ raw_schema | trim | lower }}
description: ""
tables:
- name: data__a_relation
description: ""
- name: data__b_relation
description: ""
- name: data__campaign_analytics
description: ""
{% endset %}
{{ assert_equal (actual_source_yaml | trim, expected_source_yaml | trim) }}
================================================
FILE: integration_tests/tests/test_generate_source_table_name.sql
================================================
{% set raw_schema = generate_schema_name('raw_data') %}
-- test default args
{% set actual_source_yaml = codegen.generate_source(raw_schema, name='raw') %}
{% set expected_source_yaml %}
version: 2
sources:
- name: raw
schema: {{ raw_schema | trim | lower }}
tables:
- name: data__a_relation
- name: data__b_relation
- name: data__campaign_analytics
{% endset %}
{{ assert_equal (actual_source_yaml | trim, expected_source_yaml | trim) }}
================================================
FILE: integration_tests/tests/test_generate_source_table_pattern.sql
================================================
{% set raw_schema = generate_schema_name('raw_data') %}
-- test default args
{% set actual_source_yaml = codegen.generate_source(raw_schema, table_pattern='data__b_%') %}
{% set expected_source_yaml %}
version: 2
sources:
- name: {{ raw_schema | trim | lower }}
tables:
- name: data__b_relation
{% endset %}
{{ assert_equal (actual_source_yaml | trim, expected_source_yaml | trim) }}
================================================
FILE: integration_tests/tests/test_generate_unit_test_template.sql
================================================
{% set actual_model_yaml = codegen.generate_unit_test_template(
model_name='child_model',
inline_columns=False
)
%}
-- depends_on: {{ ref('model_data_a') }}
-- depends_on: {{ ref('child_model') }}
{% set expected_model_yaml %}
unit_tests:
- name: unit_test_child_model
model: child_model
given:
- input: ref("model_data_a")
rows:
- col_a:
col_b:
expect:
rows:
- col_a:
col_b:
{% endset %}
{{ assert_equal (actual_model_yaml | trim, expected_model_yaml | trim) }}
================================================
FILE: integration_tests/tests/test_generate_unit_test_template_incremental.sql
================================================
{% set actual_model_yaml = codegen.generate_unit_test_template(
model_name='model_incremental',
)
%}
-- depends_on: {{ ref('model_incremental') }}
{% set expected_model_yaml %}
unit_tests:
- name: unit_test_model_incremental
model: model_incremental
overrides:
macros:
is_incremental: true
given:
- input: this
rows:
- id:
expect:
rows:
- id:
{% endset %}
{{ assert_equal (actual_model_yaml | trim, expected_model_yaml | trim) }}
================================================
FILE: integration_tests/tests/test_generate_unit_test_template_inline_columns.sql
================================================
{% set actual_model_yaml = codegen.generate_unit_test_template(
model_name='child_model',
inline_columns=True
)
%}
-- depends_on: {{ ref('model_data_a') }}
-- depends_on: {{ ref('child_model') }}
{% set expected_model_yaml %}
unit_tests:
- name: unit_test_child_model
model: child_model
given:
- input: ref("model_data_a")
rows:
- {col_a: , col_b: }
expect:
rows:
- {col_a: , col_b: }
{% endset %}
{{ assert_equal (actual_model_yaml | trim, expected_model_yaml | trim) }}
================================================
FILE: integration_tests/tests/test_generate_unit_test_template_model_from_source.sql
================================================
{% set actual_model_yaml = codegen.generate_unit_test_template(
model_name='model_from_source',
)
%}
-- depends_on: {{ ref('model_from_source') }}
{% set expected_model_yaml %}
unit_tests:
- name: unit_test_model_from_source
model: model_from_source
given:
- input: source("codegen_integration_tests__data_source_schema", "codegen_integration_tests__data_source_table")
rows:
- my_integer_col:
my_bool_col:
expect:
rows:
- my_integer_col:
my_bool_col:
{% endset %}
{{ assert_equal (actual_model_yaml | trim, expected_model_yaml | trim) }}
================================================
FILE: integration_tests/tests/test_generate_unit_test_template_no_inputs.sql
================================================
{% set actual_model_yaml = codegen.generate_unit_test_template(
model_name='data__a_relation',
inline_columns=False
)
%}
-- depends_on: {{ ref('data__a_relation') }}
{% set expected_model_yaml %}
unit_tests:
- name: unit_test_data__a_relation
model: data__a_relation
given: []
expect:
rows:
- col_a:
col_b:
{% endset %}
{{ assert_equal (actual_model_yaml | trim, expected_model_yaml | trim) }}
================================================
FILE: integration_tests/tests/test_helper_get_models.sql
================================================
-- depends_on: {{ ref('model_data_a') }}
-- depends_on: {{ ref('model_incremental') }}
-- depends_on: {{ ref('model_struct') }}
-- depends_on: {{ ref('model_without_import_ctes') }}
-- depends_on: {{ ref('model_without_any_ctes') }}
{% if execute %}
{% set actual_list = codegen.get_models(prefix='model_')|sort %}
{% endif %}
{% set expected_list = ['model_data_a', 'model_from_source', 'model_incremental', 'model_repeated', 'model_struct', 'model_without_any_ctes', 'model_without_import_ctes'] %}
{{ assert_equal (actual_list, expected_list) }}
================================================
FILE: macros/create_base_models.sql
================================================
{% macro create_base_models(source_name, tables) %}
{{ return(adapter.dispatch('create_base_models', 'codegen')(source_name, tables)) }}
{% endmacro %}
{% macro default__create_base_models(source_name, tables) %}
{% set source_name = ""~ source_name ~"" %}
{% set zsh_command_models = "source dbt_packages/codegen/bash_scripts/base_model_creation.sh """~ source_name ~""" " %}
{%- set models_array = [] -%}
{% for t in tables %}
{% set help_command = zsh_command_models + t %}
{{ models_array.append(help_command) }}
{% endfor %}
{{ log("Run these commands in your shell to generate the models:\n" ~ models_array|join(' && \n'), info=True) }}
{% endmacro %}
================================================
FILE: macros/generate_base_model.sql
================================================
{% macro generate_base_model(source_name, table_name, leading_commas=False, case_sensitive_cols=False, materialized=None) %}
{{ return(adapter.dispatch('generate_base_model', 'codegen')(source_name, table_name, leading_commas, case_sensitive_cols, materialized)) }}
{% endmacro %}
{% macro default__generate_base_model(source_name, table_name, leading_commas, case_sensitive_cols, materialized) %}
{%- set source_relation = source(source_name, table_name) -%}
{%- set columns = adapter.get_columns_in_relation(source_relation) -%}
{% set column_names=columns | map(attribute='name') %}
{% set base_model_sql %}
{%- if materialized is not none -%}
{{ "{{ config(materialized='" ~ materialized ~ "') }}" }}
{%- endif %}
with source as (
select * from {% raw %}{{ source({% endraw %}'{{ source_name }}', '{{ table_name }}'{% raw %}) }}{% endraw %}
),
renamed as (
select
{%- if leading_commas -%}
{%- for column in column_names %}
{{", " if not loop.first}}{% if not case_sensitive_cols %}{{ column | lower }}{% else %}{{ adapter.quote(column) }}{% endif %}
{%- endfor %}
{%- else -%}
{%- for column in column_names %}
{% if not case_sensitive_cols %}{{ column | lower }}{% else %}{{ adapter.quote(column) }}{% endif %}{{"," if not loop.last}}
{%- endfor -%}
{%- endif %}
from source
)
select * from renamed
{% endset %}
{% if execute %}
{{ print(base_model_sql) }}
{% do return(base_model_sql) %}
{% endif %}
{% endmacro %}
================================================
FILE: macros/generate_model_import_ctes.sql
================================================
{% macro generate_model_import_ctes(model_name, leading_commas = False) %}
{{ return(adapter.dispatch('generate_model_import_ctes', 'codegen')(model_name, leading_commas)) }}
{% endmacro %}
{% macro default__generate_model_import_ctes(model_name, leading_commas) %}
{%- if execute -%}
{%- set nodes = graph.nodes.values() -%}
{%- set model = (nodes
| selectattr('name', 'equalto', model_name)
| selectattr('resource_type', 'equalto', 'model')
| list).pop() -%}
{%- set model_raw_sql = model.raw_sql or model.raw_code -%}
{%- else -%}
{%- set model_raw_sql = '' -%}
{%- endif -%}
{#-
REGEX Explanations
# with_regex
- matches (start of file followed by anything then whitespace
or whitespace
or a comma) followed by the word with then a space
# from_ref
- matches (from or join) followed by some spaces and then {{ref(<something>)}}
# from_source
- matches (from or join) followed by some spaces and then {{source(<something>,<something_else>)}}
# from_var_1
- matches (from or join) followed by some spaces and then {{var(<something>)}}
# from_var_2
- matches (from or join) followed by some spaces and then {{var(<something>,<something_else>)}}
# from_table_1
- matches (from or join) followed by some spaces and then <something>.<something_else>
where each <something> is enclosed by (` or [ or " or ' or nothing)
# from_table_2
- matches (from or join) followed by some spaces and then <something>.<something_else>.<something_different>
where each <something> is enclosed by (` or [ or " or ' or nothing)
# from_table_3
- matches (from or join) followed by some spaces and then <something>
where <something> is enclosed by (` or [ or " or ')
# config block
- matches the start of the file followed by anything and then {{config(<something>)}}
-#}
{%- set re = modules.re -%}
{%- set with_regex = '(?i)(?s)(^.*\s*|\s+|,)with\s' -%}
{%- set does_raw_sql_contain_cte = re.search(with_regex, model_raw_sql) -%}
{%- set from_regexes = {
'from_ref':
'(?ix)
# first matching group
# from or join followed by at least 1 whitespace character
(from|join)\s+
# second matching group
# opening {{, 0 or more whitespace character(s), ref, 0 or more whitespace character(s), an opening parenthesis, 0 or more whitespace character(s), 1 or 0 quotation mark
({{\s*ref\s*\(\s*[\'\"]?)
# third matching group
# at least 1 of anything except a parenthesis or quotation mark
([^)\'\"]+)
# fourth matching group
# 1 or 0 quotation mark, 0 or more whitespace character(s)
([\'\"]?\s*)
# fifth matching group
# a closing parenthesis, 0 or more whitespace character(s), closing }}
(\)\s*}})
',
'from_source':
'(?ix)
# first matching group
# from or join followed by at least 1 whitespace character
(from|join)\s+
# second matching group
# opening {{, 0 or more whitespace character(s), source, 0 or more whitespace character(s), an opening parenthesis, 0 or more whitespace character(s), 1 or 0 quotation mark
({{\s*source\s*\(\s*[\'\"]?)
# third matching group
# at least 1 of anything except a parenthesis or quotation mark
([^)\'\"]+)
# fourth matching group
# 1 or 0 quotation mark, 0 or more whitespace character(s)
([\'\"]?\s*)
# fifth matching group
# a comma
(,)
# sixth matching group
# 0 or more whitespace character(s), 1 or 0 quotation mark
(\s*[\'\"]?)
# seventh matching group
# at least 1 of anything except a parenthesis or quotation mark
([^)\'\"]+)
# eighth matching group
# 1 or 0 quotation mark, 0 or more whitespace character(s)
([\'\"]?\s*)
# ninth matching group
# a closing parenthesis, 0 or more whitespace character(s), closing }}
(\)\s*}})
',
'from_var_1':
'(?ix)
# first matching group
# from or join followed by at least 1 whitespace character
(from|join)\s+
# second matching group
# opening {{, 0 or more whitespace character(s), var, 0 or more whitespace character(s), an opening parenthesis, 0 or more whitespace character(s), 1 or 0 quotation mark
({{\s*var\s*\(\s*[\'\"]?)
# third matching group
# at least 1 of anything except a parenthesis or quotation mark
([^)\'\"]+)
# fourth matching group
# 1 or 0 quotation mark, 0 or more whitespace character(s)
([\'\"]?\s*)
# fifth matching group
# a closing parenthesis, 0 or more whitespace character(s), closing }}
(\)\s*}})
',
'from_var_2':
'(?ix)
# first matching group
# from or join followed by at least 1 whitespace character
(from|join)\s+
# second matching group
# opening {{, 0 or more whitespace character(s), var, 0 or more whitespace character(s), an opening parenthesis, 0 or more whitespace character(s), 1 or 0 quotation mark
({{\s*var\s*\(\s*[\'\"]?)
# third matching group
# at least 1 of anything except a parenthesis or quotation mark
([^)\'\"]+)
# fourth matching group
# 1 or 0 quotation mark, 0 or more whitespace character(s)
([\'\"]?\s*)
# fifth matching group
# a comma
(,)
# sixth matching group
# 0 or more whitespace character(s), 1 or 0 quotation mark
(\s*[\'\"]?)
# seventh matching group
# at least 1 of anything except a parenthesis or quotation mark
([^)\'\"]+)
# eighth matching group
# 1 or 0 quotation mark, 0 or more whitespace character(s)
([\'\"]?\s*)
# ninth matching group
# a closing parenthesis, 0 or more whitespace character(s), closing }}
(\)\s*}})
',
'from_table_1':
'(?ix)
# first matching group
# from or join followed by at least 1 whitespace character
(from|join)\s+
# second matching group
# 1 or 0 of (opening bracket, backtick, or quotation mark)
([\[`\"\']?)
# third matching group
# at least 1 word character
(\w+)
# fouth matching group
# 1 or 0 of (closing bracket, backtick, or quotation mark)
([\]`\"\']?)
# fifth matching group
# a period
(\.)
# sixth matching group
# 1 or 0 of (opening bracket, backtick, or quotation mark)
([\[`\"\']?)
# seventh matching group
# at least 1 word character
(\w+)
# eighth matching group
# 1 or 0 of (closing bracket, backtick, or quotation mark) folowed by a whitespace character or end of string
([\]`\"\']?)(?=\s|$)
',
'from_table_2':
'(?ix)
# first matching group
# from or join followed by at least 1 whitespace character
(from|join)\s+
# second matching group
# 1 or 0 of (opening bracket, backtick, or quotation mark)
([\[`\"\']?)
# third matching group
# at least 1 word character
(\w+)
# fouth matching group
# 1 or 0 of (closing bracket, backtick, or quotation mark)
([\]`\"\']?)
# fifth matching group
# a period
(\.)
# sixth matching group
# 1 or 0 of (opening bracket, backtick, or quotation mark)
([\[`\"\']?)
# seventh matching group
# at least 1 word character
(\w+)
# eighth matching group
# 1 or 0 of (closing bracket, backtick, or quotation mark)
([\]`\"\']?)
# ninth matching group
# a period
(\.)
# tenth matching group
# 1 or 0 of (closing bracket, backtick, or quotation mark)
([\[`\"\']?)
# eleventh matching group
# at least 1 word character
(\w+)
# twelfth matching group
# 1 or 0 of (closing bracket, backtick, or quotation mark) folowed by a whitespace character or end of string
([\]`\"\']?)(?=\s|$)
',
'from_table_3':
'(?ix)
# first matching group
# from or join followed by at least 1 whitespace character
(from|join)\s+
# second matching group
# 1 or 0 of (opening bracket, backtick, or quotation mark)
([\[`\"\'])
# third matching group
# at least 1 word character or space
([\w ]+)
# fourth matching group
# 1 or 0 of (closing bracket, backtick, or quotation mark) folowed by a whitespace character or end of string
([\]`\"\'])(?=\s|$)
',
'config_block':'(?i)(?s)^.*{{\s*config\s*\([^)]+\)\s*}}'
} -%}
{%- set from_list = [] -%}
{%- set config_list = [] -%}
{%- set ns = namespace(model_sql = model_raw_sql) -%}
{%- for regex_name, regex_pattern in from_regexes.items() -%}
{%- set all_regex_matches = re.findall(regex_pattern, model_raw_sql) -%}
{%- for match in all_regex_matches -%}
{%- if regex_name == 'config_block' -%}
{%- set match_tuple = (match|trim, regex_name) -%}
{%- do config_list.append(match_tuple) -%}
{%- elif regex_name == 'from_source' -%}
{%- set full_from_clause = match[1:]|join|trim -%}
{%- set cte_name = 'source_' + match[6]|lower -%}
{%- set match_tuple = (cte_name, full_from_clause, regex_name) -%}
{%- do from_list.append(match_tuple) -%}
{%- elif regex_name == 'from_table_1' -%}
{%- set full_from_clause = match[1:]|join()|trim -%}
{%- set cte_name = match[2]|lower + '_' + match[6]|lower -%}
{%- set match_tuple = (cte_name, full_from_clause, regex_name) -%}
{%- do from_list.append(match_tuple) -%}
{%- elif regex_name == 'from_table_2' -%}
{%- set full_from_clause = match[1:]|join()|trim -%}
{%- set cte_name = match[2]|lower + '_' + match[6]|lower + '_' + match[10]|lower -%}
{%- set match_tuple = (cte_name, full_from_clause, regex_name) -%}
{%- do from_list.append(match_tuple) -%}
{%- else -%}
{%- set full_from_clause = match[1:]|join|trim -%}
{%- set cte_name = match[2]|trim|lower -%}
{%- set match_tuple = (cte_name, full_from_clause, regex_name) -%}
{%- do from_list.append(match_tuple) -%}
{%- endif -%}
{%- endfor -%}
{%- if regex_name == 'config_block' -%}
{%- elif regex_name == 'from_source' -%}
{%- set ns.model_sql = re.sub(regex_pattern, '\g<1> source_\g<7>', ns.model_sql) -%}
{%- elif regex_name == 'from_table_1' -%}
{%- set ns.model_sql = re.sub(regex_pattern, '\g<1> \g<3>_\g<7>', ns.model_sql) -%}
{%- elif regex_name == 'from_table_2' -%}
{%- set ns.model_sql = re.sub(regex_pattern, '\g<1> \g<3>_\g<7>_\g<11>', ns.model_sql) -%}
{%- else -%}
{%- set ns.model_sql = re.sub(regex_pattern, '\g<1> \g<3>', ns.model_sql) -%}
{% endif %}
{%- endfor -%}
{%- if from_list|length > 0 -%}
{%- set model_import_ctes -%}
{%- for config_obj in config_list -%}
{%- set ns.model_sql = ns.model_sql|replace(config_obj[0], '') -%}
{{ config_obj[0] }}
{% endfor -%}
{%- for from_obj in from_list|unique|sort -%}
{%- if loop.first -%}with {% else -%}{%- if leading_commas -%},{%- endif -%}{%- endif -%}{{ from_obj[0] }} as (
select * from {{ from_obj[1] }}
{%- if from_obj[2] == 'from_source' and from_list|length > 1 %}
-- CAUTION: It's best practice to create staging layer for raw sources
{%- elif from_obj[2] == 'from_table_1' or from_obj[2] == 'from_table_2' or from_obj[2] == 'from_table_3' %}
-- CAUTION: It's best practice to use the ref or source function instead of a direct reference
{%- elif from_obj[2] == 'from_var_1' or from_obj[2] == 'from_var_2' %}
-- CAUTION: It's best practice to use the ref or source function instead of a var
{%- endif %}
){%- if ((loop.last and does_raw_sql_contain_cte) or (not loop.last)) and not leading_commas -%},{%- endif %}
{% endfor -%}
{%- if does_raw_sql_contain_cte -%}
{%- if leading_commas -%}
{%- set replace_with = '\g<1>,' -%}
{%- else -%}
{%- set replace_with = '\g<1>' -%}
{%- endif -%}
{{ re.sub(with_regex, replace_with, ns.model_sql, 1)|trim }}
{%- else -%}
{{ ns.model_sql|trim }}
{%- endif -%}
{%- endset -%}
{%- else -%}
{% set model_import_ctes = model_raw_sql %}
{%- endif -%}
{%- if execute -%}
{{ print(model_import_ctes) }}
{% do return(model_import_ctes) %}
{% endif %}
{% endmacro %}
================================================
FILE: macros/generate_model_yaml.sql
================================================
{% macro generate_column_yaml(column, model_yaml, column_desc_dict, include_data_types, parent_column_name="") %}
{{ return(adapter.dispatch('generate_column_yaml', 'codegen')(column, model_yaml, column_desc_dict, include_data_types, parent_column_name)) }}
{% endmacro %}
{% macro default__generate_column_yaml(column, model_yaml, column_desc_dict, include_data_types, parent_column_name) %}
{% if parent_column_name %}
{% set column_name = parent_column_name ~ "." ~ column.name %}
{% else %}
{% set column_name = column.name %}
{% endif %}
{% do model_yaml.append(' - name: ' ~ column_name | lower ) %}
{% if include_data_types %}
{% do model_yaml.append(' data_type: ' ~ codegen.data_type_format_model(column)) %}
{% endif %}
{% do model_yaml.append(' description: ' ~ (column_desc_dict.get(column.name | lower,'') | tojson)) %}
{% do model_yaml.append('') %}
{% if column.fields|length > 0 %}
{% for child_column in column.fields %}
{% set model_yaml = codegen.generate_column_yaml(child_column, model_yaml, column_desc_dict, include_data_types, parent_column_name=column_name) %}
{% endfor %}
{% endif %}
{% do return(model_yaml) %}
{% endmacro %}
{% macro generate_model_yaml(model_names=[], upstream_descriptions=False, include_data_types=True) -%}
{{ return(adapter.dispatch('generate_model_yaml', 'codegen')(model_names, upstream_descriptions, include_data_types)) }}
{%- endmacro %}
{% macro default__generate_model_yaml(model_names, upstream_descriptions, include_data_types) %}
{% set model_yaml=[] %}
{% do model_yaml.append('version: 2') %}
{% do model_yaml.append('') %}
{% do model_yaml.append('models:') %}
{% if model_names is string %}
{{ exceptions.raise_compiler_error("The `model_names` argument must always be a list, even if there is only one model.") }}
{% else %}
{% for model in model_names %}
{% do model_yaml.append(' - name: ' ~ model | lower) %}
{% do model_yaml.append(' description: ""') %}
{% do model_yaml.append(' columns:') %}
{% set relation=ref(model) %}
{%- set columns = adapter.get_columns_in_relation(relation) -%}
{% set column_desc_dict = codegen.build_dict_column_descriptions(model) if upstream_descriptions else {} %}
{% for column in columns %}
{% set model_yaml = codegen.generate_column_yaml(column, model_yaml, column_desc_dict, include_data_types) %}
{% endfor %}
{% endfor %}
{% endif %}
{% if execute %}
{% set joined = model_yaml | join ('\n') %}
{{ print(joined) }}
{% do return(joined) %}
{% endif %}
{% endmacro %}
================================================
FILE: macros/generate_source.sql
================================================
{% macro get_tables_in_schema(schema_name, database_name=target.database, table_pattern='%', exclude='') %}
{% set tables=dbt_utils.get_relations_by_pattern(
schema_pattern=schema_name,
database=database_name,
table_pattern=table_pattern,
exclude=exclude
) %}
{% set table_list= tables | map(attribute='identifier') %}
{{ return(table_list | sort) }}
{% endmacro %}
{% macro generate_source(schema_name, database_name=target.database, generate_columns=False, include_descriptions=False, include_data_types=True, table_pattern='%', exclude='', name=schema_name, table_names=None, include_database=False, include_schema=False, case_sensitive_databases=False, case_sensitive_schemas=False, case_sensitive_tables=False, case_sensitive_cols=False) %}
{{ return(adapter.dispatch('generate_source', 'codegen')(schema_name, database_name, generate_columns, include_descriptions, include_data_types, table_pattern, exclude, name, table_names, include_database, include_schema, case_sensitive_databases, case_sensitive_schemas, case_sensitive_tables, case_sensitive_cols)) }}
{% endmacro %}
{% macro default__generate_source(schema_name, database_name, generate_columns, include_descriptions, include_data_types, table_pattern, exclude, name, table_names, include_database, include_schema, case_sensitive_databases, case_sensitive_schemas, case_sensitive_tables, case_sensitive_cols) %}
{% set sources_yaml=[] %}
{% do sources_yaml.append('version: 2') %}
{% do sources_yaml.append('') %}
{% do sources_yaml.append('sources:') %}
{% do sources_yaml.append(' - name: ' ~ name | lower) %}
{% if include_descriptions %}
{% do sources_yaml.append(' description: ""' ) %}
{% endif %}
{% if database_name != target.database or include_database %}
{% do sources_yaml.append(' database: ' ~ (database_name if case_sensitive_databases else database_name | lower)) %}
{% endif %}
{% if schema_name != name or include_schema %}
{% do sources_yaml.append(' schema: ' ~ (schema_name if case_sensitive_schemas else schema_name | lower)) %}
{% endif %}
{% do sources_yaml.append(' tables:') %}
{% if table_names is none %}
{% set tables=codegen.get_tables_in_schema(schema_name, database_name, table_pattern, exclude) %}
{% else %}
{% set tables = table_names %}
{% endif %}
{% for table in tables %}
{% do sources_yaml.append(' - name: ' ~ (table if case_sensitive_tables else table | lower) ) %}
{% if include_descriptions %}
{% do sources_yaml.append(' description: ""' ) %}
{% endif %}
{% if generate_columns %}
{% do sources_yaml.append(' columns:') %}
{% set table_relation=api.Relation.create(
database=database_name,
schema=schema_name,
identifier=table
) %}
{% set columns=adapter.get_columns_in_relation(table_relation) %}
{% for column in columns %}
{% do sources_yaml.append(' - name: ' ~ (column.name if case_sensitive_cols else column.name | lower)) %}
{% if include_data_types %}
{% do sources_yaml.append(' data_type: ' ~ codegen.data_type_format_source(column)) %}
{% endif %}
{% if include_descriptions %}
{% do sources_yaml.append(' description: ""' ) %}
{% endif %}
{% endfor %}
{% do sources_yaml.append('') %}
{% endif %}
{% endfor %}
{% if execute %}
{% set joined = sources_yaml | join ('\n') %}
{{ print(joined) }}
{% do return(joined) %}
{% endif %}
{% endmacro %}
================================================
FILE: macros/generate_unit_test_template.sql
================================================
{% macro generate_unit_test_template(model_name, inline_columns=false) %}
{{ return(adapter.dispatch('generate_unit_test_template', 'codegen')(model_name, inline_columns)) }}
{% endmacro %}
{% macro default__generate_unit_test_template(model_name, inline_columns=false) %}
{%- set ns = namespace(depends_on_list = []) -%}
{%- if execute -%}
-- getting inputs and materialization info
{%- for node in graph.nodes.values()
| selectattr("resource_type", "equalto", "model")
| selectattr("name", "equalto", model_name) -%}
{%- set ns.depends_on_list = ns.depends_on_list + node.depends_on.nodes -%}
{%- set ns.this_materialization = node.config['materialized'] -%}
{%- endfor -%}
{%- endif -%}
-- getting the input columns
{%- set ns.input_columns_list = [] -%}
{%- for item in ns.depends_on_list -%}
{%- set input_columns_list = [] -%}
{%- set item_dict = codegen.get_resource_from_unique_id(item) -%}
{%- if item_dict.resource_type == 'source' %}
{%- set columns = adapter.get_columns_in_relation(source(item_dict.source_name, item_dict.identifier)) -%}
{%- else -%}
{%- set columns = adapter.get_columns_in_relation(ref(item_dict.alias)) -%}
{%- endif -%}
{%- for column in columns -%}
{{ input_columns_list.append(column.name|lower) }}
{%- endfor -%}
{{ ns.input_columns_list.append(input_columns_list) }}
{%- endfor -%}
-- getting 'this' columns
{% set relation_exists = load_relation(ref(model_name)) is not none %}
{% if relation_exists %}
{%- set ns.expected_columns_list = [] -%}
{%- set columns = adapter.get_columns_in_relation(ref(model_name)) -%}
{%- for column in columns -%}
{{ ns.expected_columns_list.append(column.name|lower) }}
{%- endfor -%}
{% endif %}
{%- set unit_test_yaml_template -%}
unit_tests:
- name: unit_test_{{ model_name }}
model: {{ model_name }}
{% if ns.this_materialization == 'incremental' %}
overrides:
macros:
is_incremental: true
{% else -%}
{%- endif %}
given: {%- if ns.depends_on_list|length == 0 and ns.this_materialization != 'incremental' %} []{% endif %}
{%- for i in range(ns.depends_on_list|length) -%}
{%- set item_dict = codegen.get_resource_from_unique_id(ns.depends_on_list[i]) -%}
{% if item_dict.resource_type == 'source' %}
- input: source("{{item_dict.source_name}}", "{{item_dict.identifier}}")
rows:
{%- else %}
- input: ref("{{item_dict.alias}}")
rows:
{%- endif -%}
{%- if inline_columns -%}
{%- set ns.column_string = '- {' -%}
{%- for column_name in ns.input_columns_list[i] -%}
{%- if not loop.last -%}
{%- set ns.column_string = ns.column_string ~ column_name ~ ': , ' -%}
{%- else -%}
{%- set ns.column_string = ns.column_string ~ column_name ~ ': }' -%}
{%- endif -%}
{% endfor %}
{%- else -%}
{%- set ns.column_string = '' -%}
{%- for column_name in ns.input_columns_list[i] -%}
{%- if loop.first -%}
{%- set ns.column_string = ns.column_string ~ '- ' ~ column_name ~ ': ' -%}
{%- else -%}
{%- set ns.column_string = ns.column_string ~ '\n ' ~ column_name ~ ': ' -%}
{%- endif -%}
{% endfor %}
{%- endif %}
{{ns.column_string}}
{%- endfor %}
{%- if ns.this_materialization == 'incremental' %}
- input: this
rows:
{%- if relation_exists -%}
{%- if inline_columns -%}
{%- set ns.column_string = '- {' -%}
{%- for column_name in ns.expected_columns_list -%}
{%- if not loop.last -%}
{%- set ns.column_string = ns.column_string ~ column_name ~ ': , ' -%}
{%- else -%}
{%- set ns.column_string = ns.column_string ~ column_name ~ ': }' -%}
{%- endif -%}
{% endfor %}
{%- else -%}
{%- set ns.column_string = '' -%}
{%- for column_name in ns.expected_columns_list -%}
{%- if loop.first -%}
{%- set ns.column_string = ns.column_string ~ '- ' ~ column_name ~ ': ' -%}
{%- else -%}
{%- set ns.column_string = ns.column_string ~ '\n ' ~ column_name ~ ': ' -%}
{%- endif -%}
{% endfor %}
{%- endif %}
{{ns.column_string}}
{%- endif %}
{%- endif %}
expect:
rows:
{%- if relation_exists -%}
{%- if inline_columns -%}
{%- set ns.column_string = '- {' -%}
{%- for column_name in ns.expected_columns_list -%}
{%- if not loop.last -%}
{%- set ns.column_string = ns.column_string ~ column_name ~ ': , ' -%}
{%- else -%}
{%- set ns.column_string = ns.column_string ~ column_name ~ ': }' -%}
{%- endif -%}
{% endfor %}
{%- else -%}
{%- set ns.column_string = '' -%}
{%- for column_name in ns.expected_columns_list -%}
{%- if loop.first -%}
{%- set ns.column_string = ns.column_string ~ '- ' ~ column_name ~ ': ' -%}
{%- else -%}
{%- set ns.column_string = ns.column_string ~ '\n ' ~ column_name ~ ': ' -%}
{%- endif -%}
{% endfor %}
{%- endif %}
{{ns.column_string}}
{%- endif -%}
{%- endset -%}
{% if execute %}
{{ print(unit_test_yaml_template) }}
{% do return(unit_test_yaml_template) %}
{% endif %}
{% endmacro %}
================================================
FILE: macros/helpers/helpers.sql
================================================
{# retrieve models directly upstream from a given model #}
{% macro get_model_dependencies(model_name) %}
{% for node in graph.nodes.values() | selectattr('name', "equalto", model_name) %}
{{ return(node.depends_on.nodes) }}
{% endfor %}
{% endmacro %}
{# add to an input dictionary entries containing all the column descriptions of a given model #}
{% macro add_model_column_descriptions_to_dict(resource_type, model_name, dict_with_descriptions={}) %}
{% if resource_type == 'source' %}
{# sources aren't part of graph.nodes #}
{% set nodes = graph.sources %}
{% else %}
{% set nodes = graph.nodes %}
{% endif %}
{% for node in nodes.values()
| selectattr('resource_type', 'equalto', resource_type)
| selectattr('name', 'equalto', model_name) %}
{% for col_name, col_values in node.columns.items() %}
{% do dict_with_descriptions.update( {col_name: col_values.description} ) %}
{% endfor %}
{% endfor %}
{{ return(dict_with_descriptions) }}
{% endmacro %}
{# build a global dictionary looping through all the direct parents models #}
{# if the same column name exists with different descriptions it is overwritten at each loop #}
{% macro build_dict_column_descriptions(model_name) %}
{% if execute %}
{% set glob_dict = {} %}
{% for full_model in codegen.get_model_dependencies(model_name) %}
{% do codegen.add_model_column_descriptions_to_dict(
full_model.split('.')[0], full_model.split('.')[-1], glob_dict
) %}
{% endfor %}
{{ return(glob_dict) }}
{% endif %}
{% endmacro %}
{# build a list of models looping through all models in the project #}
{# filter by directory or prefix arguments, if provided #}
{% macro get_models(directory=None, prefix=None) %}
{% set model_names=[] %}
{% set models = graph.nodes.values() | selectattr('resource_type', "equalto", 'model') %}
{% if directory and prefix %}
{% for model in models %}
{% set model_path = "/".join(model.path.split("/")[:-1]) %}
{% if model_path == directory and model.name.startswith(prefix) %}
{% do model_names.append(model.name) %}
{% endif %}
{% endfor %}
{% elif directory %}
{% for model in models %}
{% set model_path = "/".join(model.path.split("/")[:-1]) %}
{% if model_path == directory %}
{% do model_names.append(model.name) %}
{% endif %}
{% endfor %}
{% elif prefix %}
{% for model in models if model.name.startswith(prefix) %}
{% do model_names.append(model.name) %}
{% endfor %}
{% else %}
{% for model in models %}
{% do model_names.append(model.name) %}
{% endfor %}
{% endif %}
{{ return(model_names) }}
{% endmacro %}
{% macro data_type_format_source(column) -%}
{{ return(adapter.dispatch('data_type_format_source', 'codegen')(column)) }}
{%- endmacro %}
{# format a column data type for a source #}
{% macro default__data_type_format_source(column) %}
{% set formatted = codegen.format_column(column) %}
{{ return(formatted['data_type'] | lower) }}
{% endmacro %}
{% macro data_type_format_model(column) -%}
{{ return(adapter.dispatch('data_type_format_model', 'codegen')(column)) }}
{%- endmacro %}
{# format a column data type for a model #}
{% macro default__data_type_format_model(column) %}
{% set formatted = codegen.format_column(column) %}
{{ return(formatted['data_type'] | lower) }}
{% endmacro %}
{# retrieve entire resource dictionary based on unique id #}
{% macro get_resource_from_unique_id(resource_unique_id) %}
{% set resource_type = resource_unique_id.split('.')[0] %}
{% if resource_type == 'source' %}
{% set resource = graph.sources[resource_unique_id] %}
{% elif resource_type == 'exposure' %}
{% set resource = graph.exposure[resource_unique_id] %}
{% elif resource_type == 'metric' %}
{% set resource = graph.metrics[resource_unique_id] %}
{% else %}
{% set resource = graph.nodes[resource_unique_id] %}
{% endif %}
{{ return(resource) }}
{% endmacro %}
================================================
FILE: macros/vendored/dbt_core/format_column.sql
================================================
{% macro format_column(column) -%}
{{ return(adapter.dispatch('format_column', 'codegen')(column)) }}
{%- endmacro %}
{# Vendored from: https://github.com/dbt-labs/dbt-adapters/blob/c7b12aee533184bad391a657d1753539d1dd496a/dbt/include/global_project/macros/relations/column/columns_spec_ddl.sql#L85-L89 #}
{% macro default__format_column(column) -%}
{% set data_type = column.dtype %}
{% set formatted = column.column.lower() ~ " " ~ data_type %}
{{ return({'name': column.name, 'data_type': data_type, 'formatted': formatted}) }}
{%- endmacro -%}
{# Vendored from: https://github.com/dbt-labs/dbt-bigquery/blob/4d255b2f854d21d5d8871bdaa8d7ab47e7e863a3/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql#L1-L5 #}
{# But modified to handle https://github.com/dbt-labs/dbt-codegen/issues/190 #}
{% macro bigquery__format_column(column) -%}
{% set data_type = column.data_type %}
{% if column.mode.lower() == "repeated" and column.dtype.lower() == "record" %}
{% set data_type = "array" %}
{% endif %}
{% set formatted = column.column.lower() ~ " " ~ data_type %}
{{ return({'name': column.name, 'data_type': data_type, 'formatted': formatted}) }}
{%- endmacro -%}
================================================
FILE: packages.yml
================================================
packages:
- package: dbt-labs/dbt_utils
version: [">=0.8.0", "<2.0.0"]
================================================
FILE: run_test.sh
================================================
#!/bin/bash
echo `pwd`
cd integration_tests
cp ci/sample.profiles.yml profiles.yml
dbt --warn-error deps --target $1 || exit 1
dbt --warn-error run-operation create_source_table --target $1 || exit 1
dbt --warn-error seed --target $1 --full-refresh || exit 1
dbt --warn-error run --target $1 || exit 1
dbt --warn-error test --target $1 || exit 1
================================================
FILE: supported_adapters.env
================================================
SUPPORTED_ADAPTERS=postgres,snowflake,redshift,bigquery
================================================
FILE: tox.ini
================================================
[tox]
skipsdist = True
envlist = lint_all, testenv
[testenv]
passenv =
# postgres env vars
POSTGRES_HOST
POSTGRES_USER
DBT_ENV_SECRET_POSTGRES_PASS
POSTGRES_PORT
POSTGRES_DATABASE
POSTGRES_SCHEMA
# snowflake env vars
SNOWFLAKE_ACCOUNT
SNOWFLAKE_USER
DBT_ENV_SECRET_SNOWFLAKE_PASS
SNOWFLAKE_ROLE
SNOWFLAKE_DATABASE
SNOWFLAKE_WAREHOUSE
SNOWFLAKE_SCHEMA
# bigquery env vars
BIGQUERY_PROJECT
BIGQUERY_SCHEMA
BIGQUERY_KEYFILE_JSON
# redshift env vars
REDSHIFT_HOST
REDSHIFT_USER
DBT_ENV_SECRET_REDSHIFT_PASS
REDSHIFT_DATABASE
REDSHIFT_SCHEMA
REDSHIFT_PORT
# Postgres integration tests for centralized dbt testing
# run dbt commands directly, assumes dbt is already installed in environment
[testenv:dbt_integration_postgres]
changedir = integration_tests
allowlist_externals =
dbt
skip_install = true
commands =
dbt --version
dbt --warn-error deps --target postgres
dbt --warn-error run-operation create_source_table --target postgres
dbt --warn-error seed --target postgres --full-refresh
dbt --warn-error run --target postgres
dbt --warn-error test --target postgres
# snowflake integration tests for centralized dbt testing
# run dbt commands directly, assumes dbt is already installed in environment
[testenv:dbt_integration_snowflake]
changedir = integration_tests
allowlist_externals =
dbt
skip_install = true
commands =
dbt --version
dbt --warn-error deps --target snowflake
dbt --warn-error run-operation create_source_table --target snowflake
dbt --warn-error seed --target snowflake --full-refresh
dbt --warn-error run --target snowflake
dbt --warn-error test --target snowflake
# bigquery integration tests for centralized dbt testing
# run dbt commands directly, assumes dbt is already installed in environment
[testenv:dbt_integration_bigquery]
changedir = integration_tests
allowlist_externals =
dbt
skip_install = true
commands =
dbt --version
dbt --warn-error deps --target bigquery
dbt --warn-error run-operation create_source_table --target bigquery
dbt --warn-error seed --target bigquery --full-refresh
dbt --warn-error run --target bigquery
dbt --warn-error test --target bigquery
# redshift integration tests for centralized dbt testing
# run dbt commands directly, assumes dbt is already installed in environment
[testenv:dbt_integration_redshift]
changedir = integration_tests
allowlist_externals =
dbt
skip_install = true
commands =
dbt --version
dbt --warn-error deps --target redshift
dbt --warn-error run-operation create_source_table --target redshift
dbt --warn-error seed --target redshift --full-refresh
dbt --warn-error run --target redshift
dbt --warn-error test --target redshift
gitextract_phy8nu37/ ├── .circleci/ │ └── config.yml ├── .github/ │ ├── CODEOWNERS │ ├── ISSUE_TEMPLATE/ │ │ ├── bug_report.md │ │ └── feature_request.md │ ├── pull_request_template.md │ └── workflows/ │ ├── ci.yml │ ├── stale.yml │ └── triage-labels.yml ├── .gitignore ├── CHANGELOG.md ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── README.md ├── RELEASE.md ├── bash_scripts/ │ └── base_model_creation.sh ├── dbt_project.yml ├── docker-compose.yml ├── integration_tests/ │ ├── README.md │ ├── dbt_project.yml │ ├── macros/ │ │ ├── assert_equal.sql │ │ ├── integer_type_value.sql │ │ ├── operations/ │ │ │ └── create_source_table.sql │ │ └── text_type_value.sql │ ├── models/ │ │ ├── child_model.sql │ │ ├── model_data_a.sql │ │ ├── model_from_source.sql │ │ ├── model_incremental.sql │ │ ├── model_repeated.sql │ │ ├── model_struct.sql │ │ ├── model_without_any_ctes.sql │ │ ├── model_without_import_ctes.sql │ │ ├── schema.yml │ │ └── source.yml │ ├── package-lock.yml │ ├── packages.yml │ ├── profiles.yml │ ├── seeds/ │ │ ├── data__a_relation.csv │ │ ├── data__b_relation.csv │ │ └── data__campaign_analytics.csv │ └── tests/ │ ├── test_generate_base_models.sql │ ├── test_generate_base_models_all_args.sql │ ├── test_generate_base_models_case_sensitive.sql │ ├── test_generate_base_models_leading.sql │ ├── test_generate_model_import_ctes.sql │ ├── test_generate_model_import_ctes_leading.sql │ ├── test_generate_model_import_ctes_no_ctes.sql │ ├── test_generate_model_repeated_yaml.sql │ ├── test_generate_model_struct_yaml.sql │ ├── test_generate_model_yaml.sql │ ├── test_generate_model_yaml_multiple_models.sql │ ├── test_generate_model_yaml_upstream_descriptions.sql │ ├── test_generate_model_yaml_upstream_source_descriptions.sql │ ├── test_generate_source.sql │ ├── test_generate_source_all_args.sql │ ├── test_generate_source_exclude.sql │ ├── test_generate_source_include_database_property.sql │ ├── test_generate_source_include_schema_property.sql │ ├── test_generate_source_some_tables.sql │ ├── test_generate_source_table_descriptions.sql │ ├── test_generate_source_table_name.sql │ ├── test_generate_source_table_pattern.sql │ ├── test_generate_unit_test_template.sql │ ├── test_generate_unit_test_template_incremental.sql │ ├── test_generate_unit_test_template_inline_columns.sql │ ├── test_generate_unit_test_template_model_from_source.sql │ ├── test_generate_unit_test_template_no_inputs.sql │ └── test_helper_get_models.sql ├── macros/ │ ├── create_base_models.sql │ ├── generate_base_model.sql │ ├── generate_model_import_ctes.sql │ ├── generate_model_yaml.sql │ ├── generate_source.sql │ ├── generate_unit_test_template.sql │ ├── helpers/ │ │ └── helpers.sql │ └── vendored/ │ └── dbt_core/ │ └── format_column.sql ├── packages.yml ├── run_test.sh ├── supported_adapters.env └── tox.ini
SYMBOL INDEX (1 symbols across 1 files)
FILE: integration_tests/macros/operations/create_source_table.sql
type target_schema (line 35) | create table {{ target_schema }}.codegen_integration_tests__data_source_...
Condensed preview — 80 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (151K chars).
[
{
"path": ".circleci/config.yml",
"chars": 4897,
"preview": "version: 2\n\njobs:\n\n integration-postgres:\n docker:\n - image: cimg/python:3.11\n - image: cimg/postgres:17.0"
},
{
"path": ".github/CODEOWNERS",
"chars": 31,
"preview": "* @dbt-labs/dbt-package-owners\n"
},
{
"path": ".github/ISSUE_TEMPLATE/bug_report.md",
"chars": 1434,
"preview": "---\nname: Bug report\nabout: Report a bug or an issue you've found with this package\ntitle: ''\nlabels: bug, triage\nassign"
},
{
"path": ".github/ISSUE_TEMPLATE/feature_request.md",
"chars": 813,
"preview": "---\nname: Feature request\nabout: Suggest an idea for this package\ntitle: ''\nlabels: enhancement, triage\nassignees: ''\n\n-"
},
{
"path": ".github/pull_request_template.md",
"chars": 943,
"preview": "resolves #\n\n### Problem\n\n<!---\n Describe the problem this PR is solving. What is the application state\n before this PR"
},
{
"path": ".github/workflows/ci.yml",
"chars": 1765,
"preview": "# **what?**\n# Run tests for dbt-codegen against supported adapters\n\n# **why?**\n# To ensure that dbt-codegen works as exp"
},
{
"path": ".github/workflows/stale.yml",
"chars": 786,
"preview": "# **what?**\n# For issues that have been open for awhile without activity, label\n# them as stale with a warning that they"
},
{
"path": ".github/workflows/triage-labels.yml",
"chars": 915,
"preview": "# **what?**\n# When the maintenance team triages, we sometimes need more information from the issue creator. In\n# those "
},
{
"path": ".gitignore",
"chars": 66,
"preview": "target/\ndbt_modules/\ndbt_packages/\nlogs/\nenv*/\n.venv/\n.env/\nvenv/\n"
},
{
"path": "CHANGELOG.md",
"chars": 14520,
"preview": "# dbt-codegen v0.13.1\n\n## What's Changed\n\n## Under the hood\n\n* Temporarily remove CI test for case-sensitive identifiers"
},
{
"path": "CONTRIBUTING.md",
"chars": 6403,
"preview": "# Contributing to `dbt-codegen`\n\n`dbt-codegen` is open source software. It is what it is today because community members"
},
{
"path": "LICENSE",
"chars": 11357,
"preview": " Apache License\n Version 2.0, January 2004\n "
},
{
"path": "Makefile",
"chars": 806,
"preview": ".DEFAULT_GOAL:=help\n\n.PHONY: test\ntest: ## Run the integration tests.\n\t@./run_test.sh $(target)\n\n.PHONY: test_tox\ntest: "
},
{
"path": "README.md",
"chars": 14228,
"preview": "# dbt-codegen\n\nMacros that generate dbt code, and log it to the command line.\n\n# Contents\n\n- [dbt-codegen](#dbt-codegen)"
},
{
"path": "RELEASE.md",
"chars": 1037,
"preview": "# dbt-codegen releases\n\n## When do we release?\nThere's a few scenarios that might prompt a release:\n\n| Scenario "
},
{
"path": "bash_scripts/base_model_creation.sh",
"chars": 193,
"preview": "#!/bin/bash\n\necho \"\" > models/stg_$1__$2.sql\n\ndbt --quiet run-operation codegen.generate_base_model --args '{\"source_nam"
},
{
"path": "dbt_project.yml",
"chars": 201,
"preview": "name: \"codegen\"\nversion: \"0.5.0\"\n\nrequire-dbt-version: [\">=1.1.0\", \"<3.0.0\"]\nconfig-version: 2\n\ntarget-path: \"target\"\ncl"
},
{
"path": "docker-compose.yml",
"chars": 142,
"preview": "version: \"3.7\"\nservices:\n postgres:\n image: cimg/postgres:17.0\n environment:\n - POSTGRES_USER=root\n ports"
},
{
"path": "integration_tests/README.md",
"chars": 9618,
"preview": "## Table of Contents\n\n1. [Overview](#overview)\n 1. [Prerequisites](#prerequisites)\n 2. [Introduction](#introduction)"
},
{
"path": "integration_tests/dbt_project.yml",
"chars": 465,
"preview": "name: \"codegen_integration_tests\"\nversion: \"1.0\"\nconfig-version: 2\n\nprofile: \"integration_tests\"\n\nmodel-paths: [\"models\""
},
{
"path": "integration_tests/macros/assert_equal.sql",
"chars": 483,
"preview": "{% macro assert_equal(actual_object, expected_object) %}\n{% if not execute %}\n\n {# pass #}\n\n{% elif actual_object != "
},
{
"path": "integration_tests/macros/integer_type_value.sql",
"chars": 177,
"preview": "{%- macro integer_type_value() -%}\n{%- if target.type == \"snowflake\" -%}\nnumber\n{%- elif target.type == \"bigquery\" -%}\ni"
},
{
"path": "integration_tests/macros/operations/create_source_table.sql",
"chars": 1243,
"preview": "{% macro create_source_table() %}\n\n{% set target_schema=api.Relation.create(\n database=target.database,\n schema=ta"
},
{
"path": "integration_tests/macros/text_type_value.sql",
"chars": 229,
"preview": "{%- macro text_type_value() -%}\n{%- if target.type == \"redshift\"-%}\ncharacter varying\n{%- elif target.type == \"snowflake"
},
{
"path": "integration_tests/models/child_model.sql",
"chars": 46,
"preview": "select \n * \nfrom {{ ref('model_data_a') }}\n"
},
{
"path": "integration_tests/models/model_data_a.sql",
"chars": 50,
"preview": "select \n * \nfrom {{ ref('data__a_relation') }}\n"
},
{
"path": "integration_tests/models/model_from_source.sql",
"chars": 128,
"preview": "select\n *\nfrom {{ source('codegen_integration_tests__data_source_schema', 'codegen_integration_tests__data_source_tab"
},
{
"path": "integration_tests/models/model_incremental.sql",
"chars": 63,
"preview": "{{ config(\n materialized='incremental'\n) }}\n\nselect 1 as id\n"
},
{
"path": "integration_tests/models/model_repeated.sql",
"chars": 412,
"preview": "{% if target.type == \"bigquery\" %}\n\n {#--- This exists to test the BigQuery-specific behavior requested in #190 -#}\ns"
},
{
"path": "integration_tests/models/model_struct.sql",
"chars": 546,
"preview": "{% if target.type == \"bigquery\" %}\n\n {#--- This exists to test the BigQuery-specific behavior requested in #27 -#}\n "
},
{
"path": "integration_tests/models/model_without_any_ctes.sql",
"chars": 169,
"preview": "select *, 2 as col2\nfrom {{ ref('model_without_import_ctes') }} as m\nleft join (select 2 as col_a from {{ ref('data__a_r"
},
{
"path": "integration_tests/models/model_without_import_ctes.sql",
"chars": 1459,
"preview": "/*\n This is my model!\n*/\n\n{{ config(\n materialized='table',\n) }}\n\n-- I love this cte\nwith my_first_cte as (\n se"
},
{
"path": "integration_tests/models/schema.yml",
"chars": 121,
"preview": "version: 2\n\nmodels:\n - name: model_data_a\n columns:\n - name: col_a\n description: 'description column \"a\""
},
{
"path": "integration_tests/models/source.yml",
"chars": 447,
"preview": "version: 2\n\nsources:\n - name: codegen_integration_tests__data_source_schema\n schema: \"{{ target.schema ~ '__data_sou"
},
{
"path": "integration_tests/package-lock.yml",
"chars": 122,
"preview": "packages:\n- local: ../\n- package: dbt-labs/dbt_utils\n version: 1.1.1\nsha1_hash: de2deba3d66ce03d8c02949013650cc9b94f603"
},
{
"path": "integration_tests/packages.yml",
"chars": 28,
"preview": "\npackages:\n - local: ../\n"
},
{
"path": "integration_tests/profiles.yml",
"chars": 1687,
"preview": "\n# HEY! This file is used in the dbt-codegen integrations tests with GitHub CI.\n# You should __NEVER__ check credentials"
},
{
"path": "integration_tests/seeds/data__a_relation.csv",
"chars": 20,
"preview": "col_a,col_b\n1,a\n2,b\n"
},
{
"path": "integration_tests/seeds/data__b_relation.csv",
"chars": 19,
"preview": "col_a,col_b\n3,c\n4,d"
},
{
"path": "integration_tests/seeds/data__campaign_analytics.csv",
"chars": 116,
"preview": "source,medium,source_medium,analytics,col_x\nsource_1,medium_a,1a,,x\nsource_2,medium_b,2b,,x\nsource_3,medium_c,3c,,x\n"
},
{
"path": "integration_tests/tests/test_generate_base_models.sql",
"chars": 591,
"preview": "\n{% set actual_base_model = codegen.generate_base_model(\n source_name='codegen_integration_tests__data_source_schema'"
},
{
"path": "integration_tests/tests/test_generate_base_models_all_args.sql",
"chars": 792,
"preview": "\n{% set actual_base_model = codegen.generate_base_model(\n source_name='codegen_integration_tests__data_source_schema'"
},
{
"path": "integration_tests/tests/test_generate_base_models_case_sensitive.sql",
"chars": 696,
"preview": "{% set actual_base_model = codegen.generate_base_model(\n source_name='codegen_integration_tests__data_source_schema',"
},
{
"path": "integration_tests/tests/test_generate_base_models_leading.sql",
"chars": 617,
"preview": "\n{% set actual_base_model = codegen.generate_base_model(\n source_name='codegen_integration_tests__data_source_schema'"
},
{
"path": "integration_tests/tests/test_generate_model_import_ctes.sql",
"chars": 3689,
"preview": "{% set actual_model_with_import_ctes = codegen.generate_model_import_ctes(\n model_name = 'model_without_import_ctes'\n"
},
{
"path": "integration_tests/tests/test_generate_model_import_ctes_leading.sql",
"chars": 3716,
"preview": "{% set actual_model_with_import_ctes = codegen.generate_model_import_ctes(\n model_name = 'model_without_import_ctes',"
},
{
"path": "integration_tests/tests/test_generate_model_import_ctes_no_ctes.sql",
"chars": 642,
"preview": "{% set actual_model_with_import_ctes = codegen.generate_model_import_ctes(\n model_name = 'model_without_any_ctes'\n )"
},
{
"path": "integration_tests/tests/test_generate_model_repeated_yaml.sql",
"chars": 1116,
"preview": "{% set raw_schema = generate_schema_name('raw_data') %}\n\n{% set actual_source_yaml = codegen.generate_model_yaml(\n mo"
},
{
"path": "integration_tests/tests/test_generate_model_struct_yaml.sql",
"chars": 1384,
"preview": "{% set raw_schema = generate_schema_name('raw_data') %}\n\n-- test all args\n{% set actual_source_yaml = codegen.generate_s"
},
{
"path": "integration_tests/tests/test_generate_model_yaml.sql",
"chars": 478,
"preview": "{% set actual_model_yaml = codegen.generate_model_yaml(\n model_names=['data__a_relation']\n )\n%}\n\n{% set expected_mod"
},
{
"path": "integration_tests/tests/test_generate_model_yaml_multiple_models.sql",
"chars": 588,
"preview": "{% set actual_model_yaml = codegen.generate_model_yaml(\n model_names=['data__a_relation','data__b_relation'],\n inc"
},
{
"path": "integration_tests/tests/test_generate_model_yaml_upstream_descriptions.sql",
"chars": 465,
"preview": "{% set actual_model_yaml = codegen.generate_model_yaml(\n model_names=['child_model'],\n upstream_descriptions=True,"
},
{
"path": "integration_tests/tests/test_generate_model_yaml_upstream_source_descriptions.sql",
"chars": 502,
"preview": "{% set actual_model_yaml = codegen.generate_model_yaml(\n model_names=['model_from_source'],\n upstream_descriptions"
},
{
"path": "integration_tests/tests/test_generate_source.sql",
"chars": 445,
"preview": "\n{% set raw_schema = generate_schema_name('raw_data') %}\n\n-- test default args\n{% set actual_source_yaml = codegen.gener"
},
{
"path": "integration_tests/tests/test_generate_source_all_args.sql",
"chars": 2004,
"preview": "{% set raw_schema = generate_schema_name('raw_data') %}\n\n-- test all args\n{% set actual_source_yaml = codegen.generate_s"
},
{
"path": "integration_tests/tests/test_generate_source_exclude.sql",
"chars": 459,
"preview": "\n{% set raw_schema = generate_schema_name('raw_data') %}\n\n-- test default args\n{% set actual_source_yaml = codegen.gener"
},
{
"path": "integration_tests/tests/test_generate_source_include_database_property.sql",
"chars": 498,
"preview": "\n{% set raw_schema = generate_schema_name('raw_data') %}\n\n{% set actual_source_yaml = codegen.generate_source(raw_schema"
},
{
"path": "integration_tests/tests/test_generate_source_include_schema_property.sql",
"chars": 489,
"preview": "\n{% set raw_schema = generate_schema_name('raw_data') %}\n\n{% set actual_source_yaml = codegen.generate_source(raw_schema"
},
{
"path": "integration_tests/tests/test_generate_source_some_tables.sql",
"chars": 715,
"preview": "{% set raw_schema = generate_schema_name('raw_data') %}\n\n-- test all args\n{% set actual_source_yaml = codegen.generate_s"
},
{
"path": "integration_tests/tests/test_generate_source_table_descriptions.sql",
"chars": 564,
"preview": "\n{% set raw_schema = generate_schema_name('raw_data') %}\n\n-- test default args\n{% set actual_source_yaml = codegen.gener"
},
{
"path": "integration_tests/tests/test_generate_source_table_name.sql",
"chars": 474,
"preview": "\n{% set raw_schema = generate_schema_name('raw_data') %}\n\n-- test default args\n{% set actual_source_yaml = codegen.gener"
},
{
"path": "integration_tests/tests/test_generate_source_table_pattern.sql",
"chars": 402,
"preview": "\n{% set raw_schema = generate_schema_name('raw_data') %}\n\n-- test default args\n{% set actual_source_yaml = codegen.gener"
},
{
"path": "integration_tests/tests/test_generate_unit_test_template.sql",
"chars": 557,
"preview": "{% set actual_model_yaml = codegen.generate_unit_test_template(\n model_name='child_model',\n inline_columns=False\n "
},
{
"path": "integration_tests/tests/test_generate_unit_test_template_incremental.sql",
"chars": 514,
"preview": "{% set actual_model_yaml = codegen.generate_unit_test_template(\n model_name='model_incremental',\n )\n%}\n\n-- depends_o"
},
{
"path": "integration_tests/tests/test_generate_unit_test_template_inline_columns.sql",
"chars": 540,
"preview": "{% set actual_model_yaml = codegen.generate_unit_test_template(\n model_name='child_model',\n inline_columns=True\n "
},
{
"path": "integration_tests/tests/test_generate_unit_test_template_model_from_source.sql",
"chars": 628,
"preview": "{% set actual_model_yaml = codegen.generate_unit_test_template(\n model_name='model_from_source',\n )\n%}\n\n-- depends_o"
},
{
"path": "integration_tests/tests/test_generate_unit_test_template_no_inputs.sql",
"chars": 449,
"preview": "{% set actual_model_yaml = codegen.generate_unit_test_template(\n model_name='data__a_relation',\n inline_columns=Fa"
},
{
"path": "integration_tests/tests/test_helper_get_models.sql",
"chars": 552,
"preview": "-- depends_on: {{ ref('model_data_a') }}\n-- depends_on: {{ ref('model_incremental') }}\n-- depends_on: {{ ref('model_stru"
},
{
"path": "macros/create_base_models.sql",
"chars": 677,
"preview": "{% macro create_base_models(source_name, tables) %}\n {{ return(adapter.dispatch('create_base_models', 'codegen')(sour"
},
{
"path": "macros/generate_base_model.sql",
"chars": 1527,
"preview": "{% macro generate_base_model(source_name, table_name, leading_commas=False, case_sensitive_cols=False, materialized=None"
},
{
"path": "macros/generate_model_import_ctes.sql",
"chars": 14527,
"preview": "{% macro generate_model_import_ctes(model_name, leading_commas = False) %}\n {{ return(adapter.dispatch('generate_mode"
},
{
"path": "macros/generate_model_yaml.sql",
"chars": 2792,
"preview": "{% macro generate_column_yaml(column, model_yaml, column_desc_dict, include_data_types, parent_column_name=\"\") %}\n {{ r"
},
{
"path": "macros/generate_source.sql",
"chars": 3631,
"preview": "{% macro get_tables_in_schema(schema_name, database_name=target.database, table_pattern='%', exclude='') %}\n\n {% set "
},
{
"path": "macros/generate_unit_test_template.sql",
"chars": 6126,
"preview": "{% macro generate_unit_test_template(model_name, inline_columns=false) %}\n {{ return(adapter.dispatch('generate_unit_te"
},
{
"path": "macros/helpers/helpers.sql",
"chars": 4268,
"preview": "{# retrieve models directly upstream from a given model #}\n{% macro get_model_dependencies(model_name) %}\n {% for nod"
},
{
"path": "macros/vendored/dbt_core/format_column.sql",
"chars": 1194,
"preview": "{% macro format_column(column) -%}\n {{ return(adapter.dispatch('format_column', 'codegen')(column)) }}\n{%- endmacro %}\n"
},
{
"path": "packages.yml",
"chars": 77,
"preview": "packages:\n - package: dbt-labs/dbt_utils\n version: [\">=0.8.0\", \"<2.0.0\"]\n"
},
{
"path": "run_test.sh",
"chars": 348,
"preview": "#!/bin/bash\n\necho `pwd`\ncd integration_tests\ncp ci/sample.profiles.yml profiles.yml\n\ndbt --warn-error deps --target $1 |"
},
{
"path": "supported_adapters.env",
"chars": 56,
"preview": "SUPPORTED_ADAPTERS=postgres,snowflake,redshift,bigquery\n"
},
{
"path": "tox.ini",
"chars": 2835,
"preview": "[tox]\nskipsdist = True\nenvlist = lint_all, testenv\n\n[testenv]\npassenv =\n # postgres env vars\n POSTGRES_HOST\n PO"
}
]
About this extraction
This page contains the full source code of the dbt-labs/dbt-codegen GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 80 files (137.5 KB), approximately 36.7k tokens, and a symbol index with 1 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.