Repository: microsoft/fabric-cicd Branch: main Commit: baa2832275a5 Files: 321 Total size: 1.7 MB Directory structure: gitextract_uhouros3/ ├── .changes/ │ ├── header.tpl.md │ ├── unreleased/ │ │ ├── added-20260420-140247.yaml │ │ ├── added-20260503-000000.yaml │ │ ├── fixed-20260424-103120.yaml │ │ ├── fixed-20260428-121610.yaml │ │ ├── new-items-20260505-123355.yaml │ │ └── optimization-20260505-142743.yaml │ ├── v0.1.0.md │ ├── v0.1.1.md │ ├── v0.1.10.md │ ├── v0.1.11.md │ ├── v0.1.12.md │ ├── v0.1.13.md │ ├── v0.1.14.md │ ├── v0.1.15.md │ ├── v0.1.16.md │ ├── v0.1.17.md │ ├── v0.1.18.md │ ├── v0.1.19.md │ ├── v0.1.2.md │ ├── v0.1.20.md │ ├── v0.1.21.md │ ├── v0.1.22.md │ ├── v0.1.23.md │ ├── v0.1.24.md │ ├── v0.1.25.md │ ├── v0.1.26.md │ ├── v0.1.27.md │ ├── v0.1.28.md │ ├── v0.1.29.md │ ├── v0.1.3.md │ ├── v0.1.30.md │ ├── v0.1.31.md │ ├── v0.1.32.md │ ├── v0.1.33.md │ ├── v0.1.34.md │ ├── v0.1.4.md │ ├── v0.1.5.md │ ├── v0.1.6.md │ ├── v0.1.7.md │ ├── v0.1.8.md │ ├── v0.1.9.md │ ├── v0.2.0.md │ ├── v0.3.0.md │ ├── v0.3.1.md │ └── v1.0.0.md ├── .changie.yaml ├── .github/ │ ├── CODEOWNERS │ ├── ISSUE_TEMPLATE/ │ │ ├── 1-bug.yml │ │ ├── 2-feature.yml │ │ ├── 3-documentation.yml │ │ ├── 4-question.yml │ │ └── config.yml │ ├── agents/ │ │ └── new-item-type.agent.md │ ├── copilot-instructions.md │ ├── policies/ │ │ ├── resourceManagement.yml │ │ └── sdl.yml │ ├── prompts/ │ │ ├── bug-triage.prompt.yml │ │ ├── feature-triage.prompt.yml │ │ └── question-triage.prompt.yml │ ├── pull_request_template.md │ └── workflows/ │ ├── ai-issue-triage.yml │ ├── bump.yml │ ├── changelog.yml │ ├── publish_docs.yml │ ├── test.yml │ └── validate.yml ├── .gitignore ├── .prettierignore ├── .prettierrc ├── .python-version ├── .vscode/ │ ├── extensions.json │ ├── launch.json │ └── settings.json ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── CodeQL.yml ├── LICENSE ├── README.md ├── SECURITY.md ├── activate.ps1 ├── activate.sh ├── devtools/ │ ├── debug_api.py │ ├── debug_local config.py │ ├── debug_local.py │ ├── debug_parameterization.py │ ├── debug_trace_deployment.py │ └── pypi_build_release_dev.ps1 ├── docs/ │ ├── about.md │ ├── changelog.md │ ├── code_reference.md │ ├── config/ │ │ ├── overrides/ │ │ │ └── main.html │ │ ├── pre-build/ │ │ │ ├── section_toc.py │ │ │ ├── update_item_types.py │ │ │ └── update_python_version.py │ │ └── stylesheets/ │ │ └── extra.css │ ├── example/ │ │ ├── authentication.md │ │ ├── deployment_variable.md │ │ ├── index.md │ │ └── release_pipeline.md │ ├── how_to/ │ │ ├── config_deployment.md │ │ ├── getting_started.md │ │ ├── index.md │ │ ├── item_types.md │ │ ├── optional_feature.md │ │ ├── parameterization.md │ │ └── troubleshooting.md │ └── index.md ├── mkdocs.yml ├── pyproject.toml ├── ruff.toml ├── sample/ │ └── workspace/ │ ├── ABC.Report/ │ │ ├── .platform │ │ ├── StaticResources/ │ │ │ ├── RegisteredResources/ │ │ │ │ └── test_image25861853181026917.tif │ │ │ └── SharedResources/ │ │ │ └── BaseThemes/ │ │ │ └── CY24SU10.json │ │ ├── definition.pbir │ │ └── report.json │ ├── ABC.SemanticModel/ │ │ ├── .platform │ │ ├── definition/ │ │ │ ├── cultures/ │ │ │ │ └── en-US.tmdl │ │ │ ├── database.tmdl │ │ │ ├── model.tmdl │ │ │ ├── relationships.tmdl │ │ │ └── tables/ │ │ │ ├── Table.tmdl │ │ │ └── Table_2.tmdl │ │ ├── definition.pbism │ │ └── diagramLayout.json │ ├── ABCD.Report/ │ │ ├── .platform │ │ ├── StaticResources/ │ │ │ └── SharedResources/ │ │ │ └── BaseThemes/ │ │ │ └── CY24SU10.json │ │ ├── definition.pbir │ │ └── report.json │ ├── ByConnection.Report/ │ │ ├── .platform │ │ ├── definition.pbir │ │ └── report.json │ ├── Default.Warehouse/ │ │ └── .platform │ ├── DefaultCaseInsensitive.Warehouse/ │ │ └── .platform │ ├── Example Notebook.Notebook/ │ │ ├── .platform │ │ └── notebook-content.py │ ├── Hello Copy Job.CopyJob/ │ │ ├── .platform │ │ └── copyjob-content.json │ ├── Hello Dataflow.Dataflow/ │ │ ├── .platform │ │ ├── mashup.pq │ │ └── queryMetadata.json │ ├── Hello World.Notebook/ │ │ ├── .platform │ │ └── notebook-content.py │ ├── Hello db.SQLDatabase/ │ │ ├── .gitignore │ │ ├── .platform │ │ └── Hello db.sqlproj │ ├── HelloEventhouse.Eventhouse/ │ │ ├── .children/ │ │ │ └── HelloEventhouse.KQLDatabase/ │ │ │ ├── .platform │ │ │ ├── DatabaseProperties.json │ │ │ └── DatabaseSchema.kql │ │ ├── .platform │ │ └── EventhouseProperties.json │ ├── HelloRealTimeDashboard.KQLDashboard/ │ │ ├── .platform │ │ └── RealTimeDashboard.json │ ├── MirroredDatabase_1.MirroredDatabase/ │ │ ├── .platform │ │ └── mirroring.json │ ├── OntologyDataLH.Lakehouse/ │ │ ├── .platform │ │ ├── alm.settings.json │ │ ├── lakehouse.metadata.json │ │ └── shortcuts.metadata.json │ ├── RetailSalesOntology.Ontology/ │ │ ├── .platform │ │ ├── EntityTypes/ │ │ │ ├── 205398164146535/ │ │ │ │ ├── DataBindings/ │ │ │ │ │ └── a790fdb3-e356-4f42-acf4-4420557c0fd7.json │ │ │ │ └── definition.json │ │ │ ├── 267812974919544/ │ │ │ │ ├── DataBindings/ │ │ │ │ │ └── 275d574a-4d0d-4935-9cfd-54e59ee36d7f.json │ │ │ │ └── definition.json │ │ │ ├── 28747097105824/ │ │ │ │ ├── DataBindings/ │ │ │ │ │ ├── 5f66bbb3-9bb6-415a-9cd9-9d7421d0e553.json │ │ │ │ │ └── f0767964-7f82-40a1-9ca2-f7e7fe931fcd.json │ │ │ │ └── definition.json │ │ │ └── 52068896499199/ │ │ │ ├── DataBindings/ │ │ │ │ └── 3d6fc8a5-b442-46b2-9bee-c22d08038f2e.json │ │ │ └── definition.json │ │ ├── RelationshipTypes/ │ │ │ ├── 4160405290834524422/ │ │ │ │ ├── Contextualizations/ │ │ │ │ │ └── 088a81ce-2a4c-4dbc-8887-ab6b831fa047.json │ │ │ │ └── definition.json │ │ │ ├── 4194354367812289411/ │ │ │ │ ├── Contextualizations/ │ │ │ │ │ └── 7be8afdf-2557-4950-930e-26c762fcb5a5.json │ │ │ │ └── definition.json │ │ │ └── 4244194862547506054/ │ │ │ ├── Contextualizations/ │ │ │ │ └── 7f23e2a5-25f6-4c87-8b9a-43b3b9a142ec.json │ │ │ └── definition.json │ │ └── definition.json │ ├── Run Hello World.DataPipeline/ │ │ ├── .platform │ │ ├── .schedules │ │ └── pipeline-content.json │ ├── Sample.GraphQLApi/ │ │ ├── .platform │ │ └── graphql-definition.json │ ├── SampleDataActivator.Reflex/ │ │ ├── .platform │ │ └── ReflexEntities.json │ ├── SampleDataBuildToolJob.DataBuildToolJob/ │ │ ├── .platform │ │ └── dbt-content.json │ ├── SampleEventhouse.Eventhouse/ │ │ ├── .children/ │ │ │ └── TaxiDB.KQLDatabase/ │ │ │ ├── .platform │ │ │ ├── DatabaseProperties.json │ │ │ └── DatabaseSchema.kql │ │ ├── .platform │ │ └── EventhouseProperties.json │ ├── SampleEventstream.Eventstream/ │ │ ├── .platform │ │ ├── eventstream.json │ │ └── eventstreamProperties.json │ ├── SampleKQLQueryset.KQLQueryset/ │ │ ├── .platform │ │ └── RealTimeQueryset.json │ ├── SampleSparkJobDefinition.SparkJobDefinition/ │ │ ├── .platform │ │ ├── Libs/ │ │ │ └── pipeline_config.py │ │ ├── Main/ │ │ │ └── main.py │ │ └── SparkJobDefinitionV1.json │ ├── SampleUserDataFunction.UserDataFunction/ │ │ ├── .platform │ │ ├── .resources/ │ │ │ └── functions.json │ │ ├── definition.json │ │ └── function_app.py │ ├── SourceForShortcutLH.Lakehouse/ │ │ ├── .platform │ │ ├── lakehouse.metadata.json │ │ └── shortcuts.metadata.json │ ├── TargetForShortcutLH.Lakehouse/ │ │ ├── .platform │ │ ├── lakehouse.metadata.json │ │ └── shortcuts.metadata.json │ ├── TelemetryDataEH.Eventhouse/ │ │ ├── .children/ │ │ │ └── TelemetryDataEH.KQLDatabase/ │ │ │ ├── .platform │ │ │ ├── DatabaseProperties.json │ │ │ └── DatabaseSchema.kql │ │ ├── .platform │ │ └── EventhouseProperties.json │ ├── Vars.VariableLibrary/ │ │ ├── .platform │ │ ├── settings.json │ │ ├── valueSets/ │ │ │ ├── PPE.json │ │ │ └── PROD.json │ │ └── variables.json │ ├── WithSchema.Lakehouse/ │ │ ├── .platform │ │ ├── lakehouse.metadata.json │ │ └── shortcuts.metadata.json │ ├── WithoutSchema.Lakehouse/ │ │ ├── .platform │ │ ├── lakehouse.metadata.json │ │ └── shortcuts.metadata.json │ ├── World.Environment/ │ │ ├── .platform │ │ ├── Libraries/ │ │ │ ├── CustomLibraries/ │ │ │ │ └── fabric_cicd-0.1.1-py3-none-any.whl │ │ │ └── PublicLibraries/ │ │ │ └── environment.yml │ │ └── Setting/ │ │ └── Sparkcompute.yml │ ├── cicd_experiment.MLExperiment/ │ │ ├── .platform │ │ └── mlexperiment.metadata.json │ ├── config.yml │ ├── parameter template.yml │ ├── parameter.yml │ ├── sample apache airflow job.ApacheAirflowJob/ │ │ ├── .platform │ │ ├── apacheairflowjob-content.json │ │ └── dags/ │ │ └── dag1.py │ ├── subfolder/ │ │ ├── Hello World Subfolder.Notebook/ │ │ │ ├── .platform │ │ │ └── notebook-content.py │ │ └── subfolder/ │ │ └── Hello World SubfolderSubfolder.Notebook/ │ │ ├── .platform │ │ └── notebook-content.py │ └── templates/ │ ├── nb parameter template 1.yml │ └── nb parameter template 2.yml ├── src/ │ └── fabric_cicd/ │ ├── __init__.py │ ├── _common/ │ │ ├── __init__.py │ │ ├── _check_utils.py │ │ ├── _color.py │ │ ├── _config_utils.py │ │ ├── _config_validator.py │ │ ├── _deployment_result.py │ │ ├── _exceptions.py │ │ ├── _fabric_endpoint.py │ │ ├── _file.py │ │ ├── _file_lock.py │ │ ├── _git_diff_utils.py │ │ ├── _http_tracer.py │ │ ├── _item.py │ │ ├── _logging.py │ │ ├── _validate_env_vars.py │ │ └── _validate_input.py │ ├── _items/ │ │ ├── __init__.py │ │ ├── _activator.py │ │ ├── _apacheairflowjob.py │ │ ├── _base_publisher.py │ │ ├── _copyjob.py │ │ ├── _dataagent.py │ │ ├── _databuildtooljob.py │ │ ├── _dataflowgen2.py │ │ ├── _datapipeline.py │ │ ├── _environment.py │ │ ├── _eventhouse.py │ │ ├── _eventstream.py │ │ ├── _graphqlapi.py │ │ ├── _kqldashboard.py │ │ ├── _kqldatabase.py │ │ ├── _kqlqueryset.py │ │ ├── _lakehouse.py │ │ ├── _manage_dependencies.py │ │ ├── _mirroreddatabase.py │ │ ├── _mlexperiment.py │ │ ├── _mounteddatafactory.py │ │ ├── _notebook.py │ │ ├── _ontology.py │ │ ├── _report.py │ │ ├── _semanticmodel.py │ │ ├── _sparkjobdefinition.py │ │ ├── _sqldatabase.py │ │ ├── _userdatafunction.py │ │ ├── _variablelibrary.py │ │ └── _warehouse.py │ ├── _parameter/ │ │ ├── __init__.py │ │ ├── _parameter.py │ │ └── _utils.py │ ├── constants.py │ ├── fabric_workspace.py │ └── publish.py └── tests/ ├── fixtures/ │ ├── .gitignore │ ├── README.md │ ├── __init__.py │ ├── credentials.py │ └── mock_fabric_server.py ├── test__check_utils.py ├── test__fabric_endpoint.py ├── test__file.py ├── test_config_validator.py ├── test_deploy_with_config.py ├── test_environment_publish.py ├── test_fabric_workspace.py ├── test_fqdn_workspace_id.py ├── test_git_diff_utils.py ├── test_hard_delete.py ├── test_integration_publish.py ├── test_logging.py ├── test_parameter.py ├── test_parameter_utils.py ├── test_publish.py ├── test_response_collection.py ├── test_semantic_model_exclude.py ├── test_shortcut_exclude.py ├── test_subfolders.py └── test_validate_env_vars.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .changes/header.tpl.md ================================================ # Changelog ================================================ FILE: .changes/unreleased/added-20260420-140247.yaml ================================================ kind: added body: Add `$workspace.$name` and `$workspace.$name_encoded` dynamic replacement variables for target workspace display name time: 2026-04-20T14:02:47.0254635+03:00 custom: Author: shirasassoon AuthorLink: https://github.com/shirasassoon Issue: "895" IssueLink: https://github.com/microsoft/fabric-cicd/issues/895 ================================================ FILE: .changes/unreleased/added-20260503-000000.yaml ================================================ kind: added body: Add `configure_fabric_fqdn` to configure Fabric API URLs for private-link-enabled workspaces using per-workspace FQDN endpoints time: 2026-05-03T00:00:00.0000000+00:00 custom: Author: SumeshKashyap AuthorLink: https://github.com/SumeshKashyap Issue: "754" IssueLink: https://github.com/microsoft/fabric-cicd/issues/754 ================================================ FILE: .changes/unreleased/fixed-20260424-103120.yaml ================================================ kind: fixed body: Fix incomplete Sparkcompute settings deployment by using the item definition API instead of the staging/sparkcompute API when deploying Environments time: 2026-04-24T10:31:20.3568472+02:00 custom: Author: lassevalentini AuthorLink: https://github.com/lassevalentini Issue: "776" IssueLink: https://github.com/microsoft/fabric-cicd/issues/776 ================================================ FILE: .changes/unreleased/fixed-20260428-121610.yaml ================================================ kind: fixed body: Fix HTTP 400 errors when using ``items_to_include`` with post-publish operations (e.g. Lakehouse shortcut publishing) time: 2026-04-28T12:16:10.6116726+03:00 custom: Author: shirasassoon AuthorLink: https://github.com/shirasassoon Issue: "948" IssueLink: https://github.com/microsoft/fabric-cicd/issues/948 ================================================ FILE: .changes/unreleased/new-items-20260505-123355.yaml ================================================ kind: new-items body: Add support for DataBuildToolJob item time: 2026-05-05T12:33:55.6581835-05:00 custom: Author: crazy-treyn AuthorLink: https://github.com/crazy-treyn Issue: "864" IssueLink: https://github.com/microsoft/fabric-cicd/issues/864 ================================================ FILE: .changes/unreleased/optimization-20260505-142743.yaml ================================================ kind: optimization body: Remove version check on import to reduce noise and eliminate unnecessary PyPI network call at startup time: 2026-05-05T14:27:43.6077342+03:00 custom: Author: shirasassoon AuthorLink: https://github.com/shirasassoon Issue: "973" IssueLink: https://github.com/microsoft/fabric-cicd/issues/973 ================================================ FILE: .changes/v0.1.0.md ================================================ ## [v0.1.0](https://pypi.org/project/fabric-cicd/0.1.0) - January 23, 2025 ### ✨ New Functionality - Initial public preview release - Supports Notebook, Pipeline, Semantic Model, Report, and Environment deployments - Supports User and System Identity authentication - Released to PyPi - Onboarded to Github Pages ================================================ FILE: .changes/v0.1.1.md ================================================ ## [v0.1.1](https://pypi.org/project/fabric-cicd/0.1.1) - January 23, 2025 ### 🔧 Bug Fix - Fix Environment stuck in publish ([#51](https://github.com/microsoft/fabric-cicd/issues/51)) ================================================ FILE: .changes/v0.1.10.md ================================================ ## [v0.1.10](https://pypi.org/project/fabric-cicd/0.1.10) - March 19, 2025 ### ✨ New Functionality - DataPipeline SPN Support ([#133](https://github.com/microsoft/fabric-cicd/issues/133)) ### 🔧 Bug Fix - Workspace ID replacement in data pipelines ([#164](https://github.com/microsoft/fabric-cicd/issues/164)) ### 📝 Documentation Update - Sample for passing in arguments from Azure DevOps Pipelines ================================================ FILE: .changes/v0.1.11.md ================================================ ## [v0.1.11](https://pypi.org/project/fabric-cicd/0.1.11) - March 25, 2025 ### ⚠️ Breaking Change - Parameterization refactor introducing a new parameter file structure and parameter file validation functionality ([#113](https://github.com/microsoft/fabric-cicd/issues/113)) ### ✨ New Functionality - Support regex for publish exclusion ([#121](https://github.com/microsoft/fabric-cicd/issues/121)) - Override max retries via constants ([#146](https://github.com/microsoft/fabric-cicd/issues/146)) ### 📝 Documentation Update - Update to [parameterization](https://microsoft.github.io/fabric-cicd/latest/how_to/parameterization/) docs ================================================ FILE: .changes/v0.1.12.md ================================================ ## [v0.1.12](https://pypi.org/project/fabric-cicd/0.1.12) - March 27, 2025 ### 🔧 Bug Fix - Fix constant overwrite failures ([#190](https://github.com/microsoft/fabric-cicd/issues/190)) - Fix bug where all workspace ids were not being replaced ([#186](https://github.com/microsoft/fabric-cicd/issues/186)) - Fix type hints for older versions of Python ([#156](https://github.com/microsoft/fabric-cicd/issues/156)) - Fix accepted item types constant in pre-build ================================================ FILE: .changes/v0.1.13.md ================================================ ## [v0.1.13](https://pypi.org/project/fabric-cicd/0.1.13) - April 07, 2025 ### ✨ New Functionality - Added support for Lakehouse Shortcuts - New `enable_environment_variable_replacement` feature flag ([#160](https://github.com/microsoft/fabric-cicd/issues/160)) ### 🆕 New Items Support - Onboard Workspace Folders ([#81](https://github.com/microsoft/fabric-cicd/issues/81)) - Onboard Variable Library item type ([#206](https://github.com/microsoft/fabric-cicd/issues/206)) ### ⚡ Additional Optimizations - User-agent now available in API headers ([#207](https://github.com/microsoft/fabric-cicd/issues/207)) - Fixed error log typo in fabric_endpoint ### 🔧 Bug Fix - Fix break with invalid optional parameters ([#192](https://github.com/microsoft/fabric-cicd/issues/192)) - Fix bug where all workspace ids were not being replaced by parameterization ([#186](https://github.com/microsoft/fabric-cicd/issues/186)) ================================================ FILE: .changes/v0.1.14.md ================================================ ## [v0.1.14](https://pypi.org/project/fabric-cicd/0.1.14) - April 09, 2025 ### ✨ New Functionality - Optimized & beautified terminal output - Added changelog to output of old version check ### 🔧 Bug Fix - Fix workspace folder deployments in root folder ([#221](https://github.com/microsoft/fabric-cicd/issues/221)) - Fix unpublish of workspace folders without publish ([#222](https://github.com/microsoft/fabric-cicd/issues/222)) ### ⚡ Additional Optimizations - Removed Colorama and Colorlog Dependency ================================================ FILE: .changes/v0.1.15.md ================================================ ## [v0.1.15](https://pypi.org/project/fabric-cicd/0.1.15) - April 21, 2025 ### 🔧 Bug Fix - Fix folders moving with every publish ([#236](https://github.com/microsoft/fabric-cicd/issues/236)) ### ⚡ Additional Optimizations - Introduce parallel deployments to reduce publish times ([#237](https://github.com/microsoft/fabric-cicd/issues/237)) - Improvements to check version logic ### 📝 Documentation Update - Updated Examples section in docs ================================================ FILE: .changes/v0.1.16.md ================================================ ## [v0.1.16](https://pypi.org/project/fabric-cicd/0.1.16) - April 25, 2025 ### 🔧 Bug Fix - Fix bug with folder deployment to root ([#255](https://github.com/microsoft/fabric-cicd/issues/255)) ### ⚡ Additional Optimizations - Add Workspace Name in FabricWorkspaceObject ([#200](https://github.com/microsoft/fabric-cicd/issues/200)) - New function to check SQL endpoint provision status ([#226](https://github.com/microsoft/fabric-cicd/issues/226)) ### 📝 Documentation Update - Updated Authentication docs + menu sort order ================================================ FILE: .changes/v0.1.17.md ================================================ ## [v0.1.17](https://pypi.org/project/fabric-cicd/0.1.17) - May 13, 2025 ### ⚠️ Breaking Change - Deprecate old parameter file structure ([#283](https://github.com/microsoft/fabric-cicd/issues/283)) ### 🆕 New Items Support - Onboard CopyJob item type ([#122](https://github.com/microsoft/fabric-cicd/issues/122)) - Onboard Eventstream item type ([#170](https://github.com/microsoft/fabric-cicd/issues/170)) - Onboard Eventhouse/KQL Database item type ([#169](https://github.com/microsoft/fabric-cicd/issues/169)) - Onboard Data Activator item type ([#291](https://github.com/microsoft/fabric-cicd/issues/291)) - Onboard KQL Queryset item type ([#292](https://github.com/microsoft/fabric-cicd/issues/292)) ### 🔧 Bug Fix - Fix post publish operations for skipped items ([#277](https://github.com/microsoft/fabric-cicd/issues/277)) ### ⚡ Additional Optimizations - New function `key_value_replace` for key-based replacement operations in JSON and YAML ### 📝 Documentation Update - Add publish regex example to demonstrate how to use the `publish_all_items` with regex for excluding item names ================================================ FILE: .changes/v0.1.18.md ================================================ ## [v0.1.18](https://pypi.org/project/fabric-cicd/0.1.18) - May 14, 2025 ### 🔧 Bug Fix - Fix bug with check environment publish state ([#295](https://github.com/microsoft/fabric-cicd/issues/295)) ================================================ FILE: .changes/v0.1.19.md ================================================ ## [v0.1.19](https://pypi.org/project/fabric-cicd/0.1.19) - May 21, 2025 ### 🆕 New Items Support - Onboard SQL Database item type (shell-only deployment) ([#301](https://github.com/microsoft/fabric-cicd/issues/301)) - Onboard Warehouse item type (shell-only deployment) ([#204](https://github.com/microsoft/fabric-cicd/issues/204)) ### 🔧 Bug Fix - Fix bug with unpublish workspace folders ([#273](https://github.com/microsoft/fabric-cicd/issues/273)) ================================================ FILE: .changes/v0.1.2.md ================================================ ## [v0.1.2](https://pypi.org/project/fabric-cicd/0.1.2) - January 27, 2025 ### ✨ New Functionality - Introduces max retry and backoff for long running / throttled calls ([#27](https://github.com/microsoft/fabric-cicd/issues/27)) ### 🔧 Bug Fix - Fix Environment publish uses arbitrary wait time ([#50](https://github.com/microsoft/fabric-cicd/issues/50)) - Fix Environment publish doesn't wait for success ([#56](https://github.com/microsoft/fabric-cicd/issues/56)) - Fix Long running operation steps out early for notebook publish ([#58](https://github.com/microsoft/fabric-cicd/issues/58)) ================================================ FILE: .changes/v0.1.20.md ================================================ ## [v0.1.20](https://pypi.org/project/fabric-cicd/0.1.20) - June 12, 2025 ### ✨ New Functionality - Parameterization support for find_value regex and replace_value variables ([#326](https://github.com/microsoft/fabric-cicd/issues/326)) ### 🆕 New Items Support - Onboard KQL Dashboard item type ([#329](https://github.com/microsoft/fabric-cicd/issues/329)) - Onboard Dataflow Gen2 item type ([#111](https://github.com/microsoft/fabric-cicd/issues/111)) ### 🔧 Bug Fix - Fix bug with deploying environment libraries with special chars ([#336](https://github.com/microsoft/fabric-cicd/issues/336)) ### ⚡ Additional Optimizations - Improved test coverage for subfolder creation/modification ([#211](https://github.com/microsoft/fabric-cicd/issues/211)) ================================================ FILE: .changes/v0.1.21.md ================================================ ## [v0.1.21](https://pypi.org/project/fabric-cicd/0.1.21) - June 18, 2025 ### 🔧 Bug Fix - Fix bug with workspace ID replacement in JSON files for pipeline deployments ([#345](https://github.com/microsoft/fabric-cicd/issues/345)) ### ⚡ Additional Optimizations - Increased max retry for Warehouses and Dataflows ================================================ FILE: .changes/v0.1.22.md ================================================ ## [v0.1.22](https://pypi.org/project/fabric-cicd/0.1.22) - June 25, 2025 ### 🆕 New Items Support - Onboard API for GraphQL item type ([#287](https://github.com/microsoft/fabric-cicd/issues/287)) ### 🔧 Bug Fix - Fix Fabric API call error during dataflow publish ([#352](https://github.com/microsoft/fabric-cicd/issues/352)) ### ⚡ Additional Optimizations - Expanded test coverage to handle folder edge cases ([#358](https://github.com/microsoft/fabric-cicd/issues/358)) ================================================ FILE: .changes/v0.1.23.md ================================================ ## [v0.1.23](https://pypi.org/project/fabric-cicd/0.1.23) - July 08, 2025 ### ✨ New Functionality - New functionalities for GitHub Copilot Agent and PR-to-Issue linking ### 📝 Documentation Update - Fix formatting and examples in the How to and Examples pages ### 🔧 Bug Fix - Fix issue with lakehouse shortcuts publishing ([#379](https://github.com/microsoft/fabric-cicd/issues/379)) - Add validation for empty logical IDs to prevent deployment corruption ([#86](https://github.com/microsoft/fabric-cicd/issues/86)) - Fix SQL provision print statement ([#329](https://github.com/microsoft/fabric-cicd/issues/329)) - Rename the error code for reserved item name per updated Microsoft Fabric API ([#388](https://github.com/microsoft/fabric-cicd/issues/388)) - Fix lakehouse exclude_regex to exclude shortcut publishing ([#385](https://github.com/microsoft/fabric-cicd/issues/385)) - Remove max retry limit to handle large deployments ([#299](https://github.com/microsoft/fabric-cicd/issues/299)) ================================================ FILE: .changes/v0.1.24.md ================================================ ## [v0.1.24](https://pypi.org/project/fabric-cicd/0.1.24) - August 04, 2025 ### ⚠️ Breaking Change - Require parameterization for Dataflow and Semantic Model references in Data Pipeline activities - Require specific parameterization for deploying a Dataflow that depends on another in the same workspace (see Parameterization docs) ### 📝 Documentation Update - Improve Parameterization documentation ([#415](https://github.com/microsoft/fabric-cicd/issues/415)) ### ⚡ Additional Optimizations - Support for Eventhouse query URI parameterization ([#414](https://github.com/microsoft/fabric-cicd/issues/414)) - Support for Warehouse SQL endpoint parameterization ([#392](https://github.com/microsoft/fabric-cicd/issues/392)) ### 🔧 Bug Fix - Fix Dataflow/Data Pipeline deployment failures caused by workspace permissions ([#419](https://github.com/microsoft/fabric-cicd/issues/419)) - Prevent duplicate logical ID issue in Report and Semantic Model deployment ([#405](https://github.com/microsoft/fabric-cicd/issues/405)) - Fix deployment of items without assigned capacity ([#402](https://github.com/microsoft/fabric-cicd/issues/402)) ================================================ FILE: .changes/v0.1.25.md ================================================ ## [v0.1.25](https://pypi.org/project/fabric-cicd/0.1.25) - August 19, 2025 ### ⚠️ Breaking Change - Modify the default for item_types_in_scope and add thorough validation ([#464](https://github.com/microsoft/fabric-cicd/issues/464)) ### ✨ New Functionality - Add new experimental feature flag to enable selective deployment ([#384](https://github.com/microsoft/fabric-cicd/issues/384)) - Support "ALL" environment concept in parameterization ([#320](https://github.com/microsoft/fabric-cicd/issues/320)) ### 📝 Documentation Update - Enhance Overview section in Parameterization docs ([#495](https://github.com/microsoft/fabric-cicd/issues/495)) ### ⚡ Additional Optimizations - Eliminate ACCEPTED_ITEM_TYPES_NON_UPN constant and unify with ACCEPTED_ITEM_TYPES ([#477](https://github.com/microsoft/fabric-cicd/issues/477)) - Add comprehensive GitHub Copilot instructions for effective codebase development ([#468](https://github.com/microsoft/fabric-cicd/issues/468)) ### 🔧 Bug Fix - Add feature flags and warnings for Warehouse, SQL Database, and Eventhouse unpublish operations ([#483](https://github.com/microsoft/fabric-cicd/issues/483)) - Fix code formatting inconsistencies in fabric_workspace unit test ([#474](https://github.com/microsoft/fabric-cicd/issues/474)) - Fix KeyError when deploying Reports with Semantic Model dependencies in Report-only scope case ([#278](https://github.com/microsoft/fabric-cicd/issues/278)) ================================================ FILE: .changes/v0.1.26.md ================================================ ## [v0.1.26](https://pypi.org/project/fabric-cicd/0.1.26) - September 05, 2025 ### ⚠️ Breaking Change - Deprecate Base API URL kwarg in Fabric Workspace ([#529](https://github.com/microsoft/fabric-cicd/issues/529)) ### ✨ New Functionality - Support Schedules parameterization ([#508](https://github.com/microsoft/fabric-cicd/issues/508)) - Support YAML configuration file-based deployment ([#470](https://github.com/microsoft/fabric-cicd/issues/470)) ### 📝 Documentation Update - Add dynamically generated Python version requirements to documentation ([#520](https://github.com/microsoft/fabric-cicd/issues/520)) ### ⚡ Additional Optimizations - Enhance pytest output to limit console verbosity ([#514](https://github.com/microsoft/fabric-cicd/issues/514)) ### 🔧 Bug Fix - Fix Report item schema handling ([#518](https://github.com/microsoft/fabric-cicd/issues/518)) - Fix deployment order to publish Mirrored Database before Lakehouse ([#482](https://github.com/microsoft/fabric-cicd/issues/482)) ================================================ FILE: .changes/v0.1.27.md ================================================ ## [v0.1.27](https://pypi.org/project/fabric-cicd/0.1.27) - September 05, 2025 ### 🔧 Bug Fix - Fix trailing comma in report schema ([#534](https://github.com/microsoft/fabric-cicd/issues/534)) ================================================ FILE: .changes/v0.1.28.md ================================================ ## [v0.1.28](https://pypi.org/project/fabric-cicd/0.1.28) - September 15, 2025 ### ✨ New Functionality - Add folder exclusion feature for publish operations ([#427](https://github.com/microsoft/fabric-cicd/issues/427)) - Expand workspace ID dynamic replacement capabilities in parameterization ([#408](https://github.com/microsoft/fabric-cicd/issues/408)) ### 🔧 Bug Fix - Fix unexpected behavior with file_path parameter filter ([#545](https://github.com/microsoft/fabric-cicd/issues/545)) - Fix unpublish exclude_regex bug in configuration file-based deployment ([#544](https://github.com/microsoft/fabric-cicd/issues/544)) ================================================ FILE: .changes/v0.1.29.md ================================================ ## [v0.1.29](https://pypi.org/project/fabric-cicd/0.1.29) - October 01, 2025 ### ✨ New Functionality - Support dynamic replacement for cross-workspace item IDs ([#558](https://github.com/microsoft/fabric-cicd/issues/558)) - Add option to return API response for publish operations in publish_all_items ([#497](https://github.com/microsoft/fabric-cicd/issues/497)) ### 🆕 New Items Support - Onboard Apache Airflow Job item type ([#565](https://github.com/microsoft/fabric-cicd/issues/565)) - Onboard Mounted Data Factory item type ([#406](https://github.com/microsoft/fabric-cicd/issues/406)) ### 🔧 Bug Fix - Fix publish order of Eventhouses and Semantic Models ([#566](https://github.com/microsoft/fabric-cicd/issues/566)) ================================================ FILE: .changes/v0.1.3.md ================================================ ## [v0.1.3](https://pypi.org/project/fabric-cicd/0.1.3) - January 29, 2025 ### ✨ New Functionality - Add PyPI check version to encourage version bumps ([#75](https://github.com/microsoft/fabric-cicd/issues/75)) ### 🔧 Bug Fix - Fix Semantic model initial publish results in None Url error ([#61](https://github.com/microsoft/fabric-cicd/issues/61)) - Fix Integer parsed as float failing in handle_retry for <3.12 python ([#63](https://github.com/microsoft/fabric-cicd/issues/63)) - Fix Default item types fail to unpublish ([#76](https://github.com/microsoft/fabric-cicd/issues/76)) - Fix Items in subfolders are skipped ([#77](https://github.com/microsoft/fabric-cicd/issues/77)) ### 📝 Documentation Update - Update documentation & examples ================================================ FILE: .changes/v0.1.30.md ================================================ ## [v0.1.30](https://pypi.org/project/fabric-cicd/0.1.30) - October 20, 2025 ### ✨ New Functionality - Add support for binding semantic models to on-premise gateways in Fabric workspaces ([#569](https://github.com/microsoft/fabric-cicd/issues/569)) ### 🆕 New Items Support - Add support for publishing and managing Data Agent items ([#556](https://github.com/microsoft/fabric-cicd/issues/556)) - Add OrgApp item type support ([#586](https://github.com/microsoft/fabric-cicd/issues/586)) ### ⚡ Additional Optimizations - Enhance cross-workspace variable support to allow referencing other attributes ([#583](https://github.com/microsoft/fabric-cicd/issues/583)) ### 🔧 Bug Fix - Fix workspace name extraction bug for non-ID attrs using ITEM_ATTR_LOOKUP ([#583](https://github.com/microsoft/fabric-cicd/issues/583)) - Fix capacity requirement check ([#593](https://github.com/microsoft/fabric-cicd/issues/593)) ================================================ FILE: .changes/v0.1.31.md ================================================ ## [v0.1.31](https://pypi.org/project/fabric-cicd/0.1.31) - December 01, 2025 ### ⚠️ Breaking Change - Migrate to the latest Fabric Environment item APIs to simplify deployment and improve compatibility ([#173](https://github.com/microsoft/fabric-cicd/issues/173)) ### ✨ New Functionality - Enable dynamic replacement of Lakehouse SQL Endpoint IDs ([#616](https://github.com/microsoft/fabric-cicd/issues/616)) - Enable linking of Semantic Models to both cloud and gateway connections ([#602](https://github.com/microsoft/fabric-cicd/issues/602)) - Allow use of the dynamic replacement variables within the key_value_replace parameter ([#567](https://github.com/microsoft/fabric-cicd/issues/567)) - Add support for parameter file templates ([#499](https://github.com/microsoft/fabric-cicd/issues/499)) ### 🆕 New Items Support - Add support for the ML Experiment item type ([#600](https://github.com/microsoft/fabric-cicd/issues/600)) - Add support for the User Data Function item type ([#588](https://github.com/microsoft/fabric-cicd/issues/588)) ### 📝 Documentation Update - Update the advanced Dataflow parameterization example with the correct file_path value ([#633](https://github.com/microsoft/fabric-cicd/issues/633)) ### 🔧 Bug Fix - Fix publishing issues for KQL Database items in folders ([#657](https://github.com/microsoft/fabric-cicd/issues/657)) - Separate logic for 'items to include' feature between publish and unpublish operations ([#650](https://github.com/microsoft/fabric-cicd/issues/650)) - Fix parameterization logic to properly handle find_value regex patterns and replacements ([#639](https://github.com/microsoft/fabric-cicd/issues/639)) - Correct the publish order of Data Agent and Semantic Model items ([#628](https://github.com/microsoft/fabric-cicd/issues/628)) - Fix Lakehouse item publishing errors when shortcuts refer to the default Lakehouse ID ([#610](https://github.com/microsoft/fabric-cicd/issues/610)) ================================================ FILE: .changes/v0.1.32.md ================================================ ## [v0.1.32](https://pypi.org/project/fabric-cicd/0.1.32) - December 03, 2025 ### 🔧 Bug Fix - Fix publish bug for Environment items that contain only spark settings ([#664](https://github.com/microsoft/fabric-cicd/issues/664)) ================================================ FILE: .changes/v0.1.33.md ================================================ ## [v0.1.33](https://pypi.org/project/fabric-cicd/0.1.33) - December 16, 2025 ### ✨ New Functionality - Add key_value_replace parameter support for YAML files ([#649](https://github.com/microsoft/fabric-cicd/issues/649)) - Support selective shortcut publishing with regex exclusion ([#624](https://github.com/microsoft/fabric-cicd/issues/624)) ### ⚡ Additional Optimizations - Add Linux development environment bootstrapping script ([#680](https://github.com/microsoft/fabric-cicd/issues/680)) - Update item types in scope to be an optional parameter in validate parameter file function ([#669](https://github.com/microsoft/fabric-cicd/issues/669)) ### 🔧 Bug Fix - Fix publish order for Notebook and Eventhouse dependent items ([#685](https://github.com/microsoft/fabric-cicd/issues/685)) - Enable parameterizing multiple connections in the same Semantic Model item ([#674](https://github.com/microsoft/fabric-cicd/issues/674)) - Fix missing description metadata in item payload for shell-only item deployments ([#672](https://github.com/microsoft/fabric-cicd/issues/672)) - Resolve API long running operation handling when publishing Environment items ([#668](https://github.com/microsoft/fabric-cicd/issues/668)) ================================================ FILE: .changes/v0.1.34.md ================================================ ## [v0.1.34](https://pypi.org/project/fabric-cicd/0.1.34) - January 20, 2026 ### ✨ New Functionality - Enable dynamic replacement of SQL endpoint values from SQL Database items ([#720](https://github.com/microsoft/fabric-cicd/issues/720)) - Support Fabric Notebook Authentication ([#707](https://github.com/microsoft/fabric-cicd/issues/707)) ### 🆕 New Items Support - Onboard Spark Job Definition item type ([#115](https://github.com/microsoft/fabric-cicd/issues/115)) ### 📝 Documentation Update - Add `CONTRIBUTING.md` file to repository ([#723](https://github.com/microsoft/fabric-cicd/issues/723)) - Add comprehensive troubleshooting guide to documentation ([#705](https://github.com/microsoft/fabric-cicd/issues/705)) - Add parameterization documentation for Report items using ByConnection binding to Semantic Models ([#637](https://github.com/microsoft/fabric-cicd/issues/637)) ### ⚡ Additional Optimizations - Add debug file for local Fabric REST API testing ([#714](https://github.com/microsoft/fabric-cicd/issues/714)) ================================================ FILE: .changes/v0.1.4.md ================================================ ## [v0.1.4](https://pypi.org/project/fabric-cicd/0.1.4) - February 12, 2025 ### ✨ New Functionality - Support Feature Flagging ([#96](https://github.com/microsoft/fabric-cicd/issues/96)) ### 🔧 Bug Fix - Fix Image support in report deployment ([#88](https://github.com/microsoft/fabric-cicd/issues/88)) - Fix Broken README link ([#92](https://github.com/microsoft/fabric-cicd/issues/92)) ### ⚡ Additional Optimizations - Workspace ID replacement improved - Increased error handling in activate script - Onboard pytest and coverage - Improvements to nested dictionaries ([#37](https://github.com/microsoft/fabric-cicd/issues/37)) - Support Python Installed From Windows Store ([#87](https://github.com/microsoft/fabric-cicd/issues/87)) ================================================ FILE: .changes/v0.1.5.md ================================================ ## [v0.1.5](https://pypi.org/project/fabric-cicd/0.1.5) - February 18, 2025 ### 🔧 Bug Fix - Fix Environment Failure without Public Library ([#103](https://github.com/microsoft/fabric-cicd/issues/103)) ### ⚡ Additional Optimizations - Introduces pytest check for PRs ([#100](https://github.com/microsoft/fabric-cicd/issues/100)) ================================================ FILE: .changes/v0.1.6.md ================================================ ## [v0.1.6](https://pypi.org/project/fabric-cicd/0.1.6) - February 24, 2025 ### 🆕 New Items Support - Onboard Lakehouse item type ([#116](https://github.com/microsoft/fabric-cicd/issues/116)) ### 📝 Documentation Update - Update example docs ([#25](https://github.com/microsoft/fabric-cicd/issues/25)) - Update find_replace docs ([#110](https://github.com/microsoft/fabric-cicd/issues/110)) ### ⚡ Additional Optimizations - Standardized docstrings to Google format - Onboard file objects ([#46](https://github.com/microsoft/fabric-cicd/issues/46)) - Leverage UpdateDefinition Flag ([#28](https://github.com/microsoft/fabric-cicd/issues/28)) - Convert repo and workspace dictionaries ([#45](https://github.com/microsoft/fabric-cicd/issues/45)) ================================================ FILE: .changes/v0.1.7.md ================================================ ## [v0.1.7](https://pypi.org/project/fabric-cicd/0.1.7) - February 26, 2025 ### 🔧 Bug Fix - Fix special character support in files ([#129](https://github.com/microsoft/fabric-cicd/issues/129)) ================================================ FILE: .changes/v0.1.8.md ================================================ ## [v0.1.8](https://pypi.org/project/fabric-cicd/0.1.8) - March 04, 2025 ### 🔧 Bug Fix - Handle null byPath object in report definition file ([#143](https://github.com/microsoft/fabric-cicd/issues/143)) - Support relative directories ([#136](https://github.com/microsoft/fabric-cicd/issues/136)) ([#132](https://github.com/microsoft/fabric-cicd/issues/132)) - Increase special character support ([#134](https://github.com/microsoft/fabric-cicd/issues/134)) ### ⚡ Additional Optimizations - Changelog now available with version check ([#127](https://github.com/microsoft/fabric-cicd/issues/127)) ================================================ FILE: .changes/v0.1.9.md ================================================ ## [v0.1.9](https://pypi.org/project/fabric-cicd/0.1.9) - March 11, 2025 ### 🆕 New Items Support - Support for Mirrored Database item type ([#145](https://github.com/microsoft/fabric-cicd/issues/145)) ### ⚡ Additional Optimizations - Increase reserved name wait time ([#135](https://github.com/microsoft/fabric-cicd/issues/135)) ================================================ FILE: .changes/v0.2.0.md ================================================ ## [v0.2.0](https://pypi.org/project/fabric-cicd/0.2.0) - February 16, 2026 ### ✨ New Functionality - Support parallelize deployments within a given item type by [mdrakiburrahman](https://github.com/mdrakiburrahman) ([#719](https://github.com/microsoft/fabric-cicd/issues/719)) - Add a black-box REST API testing harness by [mdrakiburrahman](https://github.com/mdrakiburrahman) ([#738](https://github.com/microsoft/fabric-cicd/issues/738)) - Change header print messages to info log by [mwc360](https://github.com/mwc360) ([#771](https://github.com/microsoft/fabric-cicd/issues/771)) - Add support for semantic model binding per environment by [shirasassoon](https://github.com/shirasassoon) ([#689](https://github.com/microsoft/fabric-cicd/issues/689)) ### 🔧 Bug Fix - Remove OrgApp item type support by [shirasassoon](https://github.com/shirasassoon) ([#758](https://github.com/microsoft/fabric-cicd/issues/758)) - Improve environment-mapping behavior in optional config fields by [shirasassoon](https://github.com/shirasassoon) ([#716](https://github.com/microsoft/fabric-cicd/issues/716)) - Fix duplicate YAML key detection in parameter validation by [shirasassoon](https://github.com/shirasassoon) ([#752](https://github.com/microsoft/fabric-cicd/issues/752)) - Add caching for item attribute lookups by [MiSchroe](https://github.com/MiSchroe) ([#704](https://github.com/microsoft/fabric-cicd/issues/704)) ### ⚡ Additional Optimizations - Enable configuration-based deployment without feature flags by [shirasassoon](https://github.com/shirasassoon) ([#805](https://github.com/microsoft/fabric-cicd/issues/805)) ### 📝 Documentation Update - Fix troubleshooting docs by [shirasassoon](https://github.com/shirasassoon) ([#747](https://github.com/microsoft/fabric-cicd/issues/747)) ================================================ FILE: .changes/v0.3.0.md ================================================ ## [v0.3.0](https://pypi.org/project/fabric-cicd/0.3.0) - March 09, 2026 ### ✨ New Functionality - Support selective folder deployment using inclusion list by [shirasassoon](https://github.com/shirasassoon) ([#757](https://github.com/microsoft/fabric-cicd/issues/757)) - Add FABRIC_CICD_VERSION_CHECK_DISABLED environment variable to disable version check on startup by [Ricapar](https://github.com/Ricapar) ([#811](https://github.com/microsoft/fabric-cicd/issues/811)) - Add enhanced logging configuration options via public functions by [shirasassoon](https://github.com/shirasassoon) ([#842](https://github.com/microsoft/fabric-cicd/issues/842)) - Add deployment support for Notebook items with `.ipynb` file format by [shirasassoon](https://github.com/shirasassoon) ([#850](https://github.com/microsoft/fabric-cicd/issues/850)) ### 🔧 Bug Fix - Add max workers soft cap with override and handle garbage response from Fabric Data Plane by [mdrakiburrahman](https://github.com/mdrakiburrahman) ([#827](https://github.com/microsoft/fabric-cicd/issues/827)) - Fix parameter validation failure for item names with accented characters by [shirasassoon](https://github.com/shirasassoon) ([#818](https://github.com/microsoft/fabric-cicd/issues/818)) - Exclude items with placeholder logical ID from duplicate logical ID check by [shirasassoon](https://github.com/shirasassoon) ([#843](https://github.com/microsoft/fabric-cicd/issues/843)) ### ⚡ Additional Optimizations - Add return value to deploy_with_config by [ayeshurun](https://github.com/ayeshurun) ([#851](https://github.com/microsoft/fabric-cicd/issues/851)) - Add support for Python 3.13 in the library by [shirasassoon](https://github.com/shirasassoon) ([#855](https://github.com/microsoft/fabric-cicd/issues/855)) ### 📝 Documentation Update - Remove incorrect statement on support for parameter replacements in Platform files by [shirasassoon](https://github.com/shirasassoon) ([#839](https://github.com/microsoft/fabric-cicd/issues/839)) ================================================ FILE: .changes/v0.3.1.md ================================================ ## [v0.3.1](https://pypi.org/project/fabric-cicd/0.3.1) - March 12, 2026 ### 🔧 Bug Fix - Fix override behavior of feature flags and constants in config deployment by [shirasassoon](https://github.com/shirasassoon) ([#872](https://github.com/microsoft/fabric-cicd/issues/872)) ================================================ FILE: .changes/v1.0.0.md ================================================ ## [v1.0.0](https://pypi.org/project/fabric-cicd/1.0.0) - April 20, 2026 ### ⚠️ Breaking Change - Remove default credential fallback; explicit token credential is now required by [shirasassoon](https://github.com/shirasassoon) ([#909](https://github.com/microsoft/fabric-cicd/issues/909)) - Remove implicit authentication in Microsoft Fabric Notebook and identity logging; require explicit token credential and keyword-only arguments for `FabricWorkspace` and `deploy_with_config` by [shirasassoon](https://github.com/shirasassoon) ([#930](https://github.com/microsoft/fabric-cicd/issues/930)) ### 🆕 New Items Support - Add support for Ontology item type by [shirasassoon](https://github.com/shirasassoon) ([#796](https://github.com/microsoft/fabric-cicd/issues/796)) ### ✨ New Functionality - Improve transparency of `deploy_with_config` function in success and failure scenarios by [shirasassoon](https://github.com/shirasassoon) ([#695](https://github.com/microsoft/fabric-cicd/issues/695)) - Extend API response collection to unpublish operations by [shirasassoon](https://github.com/shirasassoon) ([#877](https://github.com/microsoft/fabric-cicd/issues/877)) - Add `get_changed_items()` utility function to detect Fabric items changed via git diff for use with selective deployment by [vipulb91](https://github.com/vipulb91) ([#865](https://github.com/microsoft/fabric-cicd/issues/865)) ### 🔧 Bug Fix - Prevent unintended GUID replacements in Variable Library item files during publish by [shirasassoon](https://github.com/shirasassoon) ([#884](https://github.com/microsoft/fabric-cicd/issues/884)) - Fix YAML content check to reject notebook and other non-YAML files during `key_value_replace` parameterization, preventing file corruption by [shirasassoon](https://github.com/shirasassoon) ([#890](https://github.com/microsoft/fabric-cicd/issues/890)) - Fix Notebook deployment failure caused by non-deterministic ordering of definition files in API payload by [shirasassoon](https://github.com/shirasassoon) ([#869](https://github.com/microsoft/fabric-cicd/issues/869)) - Ignore parameter file when not explicitly defined in config file by [aviatco](https://github.com/aviatco) ([#866](https://github.com/microsoft/fabric-cicd/issues/866)) - Add `enable_hard_delete` feature flag to bypass workspace recycle bin during unpublish by [shirasassoon](https://github.com/shirasassoon) ([#924](https://github.com/microsoft/fabric-cicd/issues/924)) - Add timeout for long-running operation polling by [shirasassoon](https://github.com/shirasassoon) ([#919](https://github.com/microsoft/fabric-cicd/issues/919)) ================================================ FILE: .changie.yaml ================================================ # docs: https://changie.dev/config/ --- changesDir: .changes unreleasedDir: unreleased headerPath: header.tpl.md versionExt: md changelogPath: docs/changelog.md versionFormat: '## [{{.Version}}](https://pypi.org/project/fabric-cicd/{{.Version}}) - {{.Time.Format "January 02, 2006"}}' kindFormat: "### {{.Kind}}" changeFormat: "* {{.Body}} by [{{.Custom.Author}}]({{.Custom.AuthorLink}}) ([#{{.Custom.Issue}}]({{.Custom.IssueLink}}))" kinds: - label: "⚠️ Breaking Change" key: breaking auto: major - label: "🆕 New Items Support" key: new-items auto: minor - label: "✨ New Functionality" key: added auto: minor - label: "🔧 Bug Fix" key: fixed auto: patch - label: "⚡ Additional Optimizations" key: optimization auto: patch - label: "📝 Documentation Update" key: docs auto: patch newlines: afterChangelogHeader: 0 beforeChangelogVersion: 1 endOfVersion: 1 afterKind: 1 beforeKind: 1 envPrefix: CHANGIE_ custom: - key: Issue label: Issue Number type: int minLength: 1 - key: Author label: Author's GitHub Username type: string minLength: 3 post: - key: AuthorLink value: "https://github.com/{{.Custom.Author}}" - key: IssueLink value: "https://github.com/microsoft/fabric-cicd/issues/{{.Custom.Issue}}" replacements: - path: src/fabric_cicd/constants.py find: 'VERSION = ".*"' replace: 'VERSION = "{{.VersionNoPrefix}}"' ================================================ FILE: .github/CODEOWNERS ================================================ ########################################## # CODE OWNERS ########################################## # default ownership: default owners for everything in the repo (Unless a later match takes precedence) - @microsoft/fabric-cicd ================================================ FILE: .github/ISSUE_TEMPLATE/1-bug.yml ================================================ name: "🐛 Bug Report" description: Report a bug title: "[BUG] " labels: ["bug"] body: - type: markdown attributes: value: Thanks for taking the time to report a bug! Please fill out the information below to help us diagnose and fix the issue. - type: input id: library-version attributes: label: Library Version description: What is the library version? validations: required: true - type: input id: python-version attributes: label: Python Version description: Run `python --version` to get your version validations: required: true - type: dropdown id: operating-system attributes: label: Operating System description: What operating system are you using? options: - Windows - macOS - Linux validations: required: true - type: dropdown id: auth-method attributes: label: Authentication Method description: How are you authenticating? options: - Azure CLI (az login) - Service principal (secret) - Service principal (certificate) - Service principal (federated credential) - Managed identity - Other validations: required: true - type: textarea id: problem-description attributes: label: What is the problem? description: Describe the bug you encountered and what the gap is. validations: required: true - type: textarea id: reproduction-steps attributes: label: Steps to reproduce description: Please provide step-by-step instructions to reproduce the bug validations: required: true - type: textarea id: expected-behavior attributes: label: Expected behavior description: What did you expect to happen? validations: required: true - type: textarea id: actual-behavior attributes: label: Actual behavior description: What actually happened? Include error messages if applicable. validations: required: true - type: textarea id: solution-description attributes: label: Is there an ideal solution? description: Describe the ideal solution if there is one. validations: required: false - type: textarea id: additional-context attributes: label: Additional context, screenshots, logs, error output, etc description: Add any other relevant details like specific workspace/item types involved, related issues or documentation, API request IDs. If it is long, please paste to https://gist.github.com/ and insert the link here. Ensure to strip any non public information. validations: required: false ================================================ FILE: .github/ISSUE_TEMPLATE/2-feature.yml ================================================ name: "🚀 Feature Request" description: Suggest an idea or enhancement for fabric-cicd title: "[FEATURE] " labels: ["enhancement"] body: - type: markdown attributes: value: | Thanks for suggesting a new feature! Please provide as much detail as possible to help us understand and evaluate your request. - type: textarea id: use-case-problem attributes: label: Use Case / Problem description: | Describe the problem or use case that this feature would solve. Include: - What problem are you trying to solve? - Current limitations or pain points - How this fits into your workflow validations: required: true - type: textarea id: proposed-solution attributes: label: Proposed Solution description: | Describe the solution you'd like to see implemented: - Specific functionality - Expected behavior and output - Integration with existing fabric-cicd features validations: required: true - type: textarea id: alternatives-considered attributes: label: Alternatives Considered description: | Describe any alternative solutions or features you've considered: - Workarounds you're currently using - Other tools or approaches that could solve this - Why those alternatives are insufficient - type: checkboxes id: impact-assessment attributes: label: Impact Assessment description: Help us understand the impact of this feature options: - label: This would help me personally - label: This would help my team/organization - label: This would help the broader fabric-cicd community - label: This aligns with Microsoft Fabric roadmap items - type: checkboxes id: implementation-attestation attributes: label: Implementation Attestation description: Please confirm your understanding of implementation considerations (required) options: - label: I understand this feature should maintain backward compatibility (if applicable) required: true - label: I understand this feature should not introduce performance regressions for existing workflows required: true - label: I acknowledge that new features must follow fabric-cicd's established patterns and conventions required: true - type: textarea id: implementation-notes attributes: label: Implementation Notes description: | If you have technical suggestions for implementation, consider: - Which item types are affected? (Notebook, DataPipeline, Environment, etc.) - Does this require changes to core classes/functions? Which ones? - Does this require changes to the parameterization framework? - Which Fabric REST APIs would be involved (if any)? ================================================ FILE: .github/ISSUE_TEMPLATE/3-documentation.yml ================================================ name: "📝 Documentation Feedback" description: Provide documentation feedback title: "[DOCUMENTATION] " labels: ["documentation"] body: - type: markdown attributes: value: Thanks for taking the time to provide documentation feedback! Please include as much detail as possible to help us improve our documentation. - type: input id: documentation-location attributes: label: URL to documentation description: Provide a URL to the documentation location. placeholder: "https://microsoft.github.io/fabric-cicd/..." validations: required: true - type: dropdown id: feedback-type attributes: label: Type of feedback description: What kind of documentation issue is this? options: - Unclear or confusing content - Missing information - Incorrect or outdated information - Typo or grammatical error - Code example issue - Broken link - Suggestion for improvement - Other validations: required: true - type: textarea id: current-content attributes: label: Current content description: Copy the relevant section of documentation that needs improvement (if applicable). placeholder: "Paste the current text or describe what exists..." validations: required: false - type: textarea id: feedback attributes: label: Feedback description: What is the issue or what improvement would you suggest? placeholder: "Describe what's wrong or what could be better..." validations: required: true - type: textarea id: suggested-change attributes: label: Suggested change description: If you have a specific suggestion, please provide it here. placeholder: "Provide your suggested text, code, or improvement..." validations: required: false - type: dropdown id: user-experience attributes: label: Experience with fabric-cicd description: This helps us understand our audience better. options: - New to fabric-cicd - Some experience with fabric-cicd - Experienced user validations: required: false - type: textarea id: additional-context attributes: label: Additional context description: Add any other context, screenshots, or related links here. validations: required: false ================================================ FILE: .github/ISSUE_TEMPLATE/4-question.yml ================================================ name: "❓ Question" description: Ask a question about fabric-cicd title: "[QUESTION] " labels: ["question"] body: - type: markdown attributes: value: Thanks for your question! Please provide as much detail as possible to help us assist you effectively. - type: textarea id: question attributes: label: What is the question? description: A clear and concise description of what you'd like to know or need help with. validations: required: true - type: textarea id: context attributes: label: Context description: | Add context and include: - Your use case or scenario - What you've already tried - Specifics on what was involved (code snippets, configurations, etc.) validations: required: true - type: input id: library-version attributes: label: Library Version description: What is the library version? (if relevant) - type: dropdown id: operating-system attributes: label: Operating System description: What operating system are you using? (if relevant) options: - Not applicable - Windows - macOS - Linux default: 0 - type: checkboxes id: documentation-check attributes: label: Have you checked the documentation? description: Please confirm you've reviewed the available resources options: - label: I have searched existing GitHub issues required: true - label: I have reviewed the fabric-cicd documentation required: true - type: textarea id: additional-information attributes: label: Additional Information description: | Any additional details that might help us answer your question: - Code snippets - Screenshots (if applicable) - Error messages (if any) - type: markdown attributes: value: | ## Alternative Support Channels For different types of questions, you might also consider: - **General Fabric questions**: [Developer Community Forum](https://community.fabric.microsoft.com/t5/Developer/bd-p/Developer) - **Feature suggestions**: [Fabric Ideas Portal](https://ideas.fabric.microsoft.com/) - **Enterprise support**: [Fabric Support Team](https://support.fabric.microsoft.com/) - **Community discussions**: [r/MicrosoftFabric on Reddit](https://www.reddit.com/r/MicrosoftFabric/) ================================================ FILE: .github/ISSUE_TEMPLATE/config.yml ================================================ blank_issues_enabled: false ================================================ FILE: .github/agents/new-item-type.agent.md ================================================ --- name: New Item Type description: Guide and assist with onboarding a new Microsoft Fabric item type into fabric-cicd argument-hint: Tell me which Fabric item type you want to add (e.g., "Add support for Ontology") tools: - runInTerminal - terminalLastCommand - search - fetch - readFile - editFiles - createFile --- # New Item Type Onboarding Agent > **Important:** If you are unsure about any detail — such as whether the item type supports definitions, has unique deployment requirements, depends on other item types, or requires parameterization — **always ask the requestor for clarification before proceeding**. List the specific unknowns and ask the requestor to confirm each one before continuing. ## Prerequisites Before starting, you need only the **display name** (PascalCase, as used by the Fabric API — e.g., `CopyJob`, `SnowflakeDatabase`). The eligibility gates below will determine the remaining core details. ### Eligibility Gates Before proceeding, confirm all of the following. If any gate fails, **stop** — the item type cannot be onboarded. 1. The item type must be supported in source control / Git integration. Fetch the [item definition overview](https://learn.microsoft.com/en-us/rest/api/fabric/articles/item-management/definitions/item-definition-overview) page and check if the item type appears in the **"Definition Details for Supported Item Types"** list. 2. The Fabric API must support deployment for the item type — either full definition deployment or shell-only creation (like Lakehouse/Warehouse). Verify using the steps in **"How to verify gates 2 and 3"** below. 3. The Fabric API must support service principal (SPN) authentication for the item type's deployment operations. fabric-cicd is primarily used in CI/CD pipelines where SPN is the standard authentication method. Verify using the steps in **"How to verify gates 2 and 3"** below. #### How to verify gates 2 and 3 The Fabric REST API docs follow a predictable URL pattern. Use these steps to directly navigate to the right pages instead of searching the generic API root: 1. **Construct the item type's API items page URL:** `https://learn.microsoft.com/en-us/rest/api/fabric/{itemtype-lowercase}/items` where `{itemtype-lowercase}` is the PascalCase item type name lowercased with no separators. - Examples: `SnowflakeDatabase` → `snowflakedatabase`, `DataPipeline` → `datapipeline`, `KQLDatabase` → `kqldatabase` 2. **Fetch that page and confirm deployment operations exist (Gate 2):** - Full definition support: Look for **"Get [Item] Definition"** and **"Update [Item] Definition"** operations in the Operations table. - Shell-only support: If the documentation explicitly states that item definition is not supported (e.g., Warehouse's Create page states "This API does not support item definition") and the item has **"Create [Item]"** and **"Update [Item]"** operations, the item is shell-only. **Important:** There can be nuances with definition support — always present your findings to the requestor and ask them to confirm whether the item should be categorized as full definition or shell-only before proceeding. - If neither Create nor definition operations exist, Gate 2 fails. 3. **Fetch the Create endpoint page to check SPN support (Gate 3):** `https://learn.microsoft.com/en-us/rest/api/fabric/{itemtype-lowercase}/items/create-{item-type-kebab-case}` where `{item-type-kebab-case}` inserts hyphens between PascalCase words and lowercases everything. - Examples: `SnowflakeDatabase` → `create-snowflake-database`, `DataPipeline` → `create-data-pipeline`, `KQLDatabase` → `create-kql-database` - Find the **"Microsoft Entra supported identities"** section and confirm the table shows **"Service principal and Managed identities: Yes"**. If it shows "No" or the section is missing, Gate 3 fails. **Important:** You MUST fetch the actual API pages listed above. Do not guess or assume gate results based on the item type name alone. If a URL returns a 404, try alternate kebab-case patterns (e.g., `graphqlapi` instead of `graph-q-l-api`), or fall back to fetching the [Fabric REST API root](https://learn.microsoft.com/en-us/rest/api/fabric/) and navigating to the item type's section. If the documentation is still inaccessible, ask the requestor to provide the relevant API page details. **Exceptions:** Gates 1 and 3 may be excepted on a case-by-case basis with fabric-cicd team approval — for example, Notebook `.ipynb` format is supported despite not being source-controlled (gate 1 exception). Gate 2 (API deployment support) has no exceptions. Any approved exception must be documented as a known limitation in `docs/how_to/item_types.md`. ### Additional Details (Required) After the eligibility gates pass, you **must** ask the requestor about **every row** in this table and record the answer before starting implementation. Do not skip any row or assume the answer is "no." | Question to ask the requestor | Example | Affects | | ----------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | ----------- | | Which existing item types must deploy **before** this one? Which existing types depend on this one deploying first? | Eventhouse → KQLDatabase, SemanticModel → Report | Step 1b | | Should certain file paths within the item folder be skipped during publish? | `.pbi/`, `.children/` | Step 1d | | Does the API use a non-standard definition format? If yes, provide the exact format string and instructions for how it should be handled. | `ipynb`, `SparkJobDefinitionV2` | Step 1e | | Does deleting this item destroy user data? | Lakehouse, Eventhouse | Step 1f | | Can items of this type reference each other? | Data Pipeline invokes another Data Pipeline | Steps 1g, 2 | | Does the item need custom deployment logic, dependency resolution, or special parameterization? If yes, describe what is needed. | Lakehouse creation payload, Environment async check, KQL items need query service URI (via dynamic replacement), SemanticModel binding parameter | Step 2 | --- ## Safety Rules - **Never hardcode secrets, tokens, or credentials** in publisher code or tests - **Use deterministic test data** — no real tenant IDs, workspace IDs, or user emails - **Follow existing patterns** — consistency is more important than cleverness - **Validate all assumptions** — if unsure about API behavior, ask the requestor --- ## Integration Checklist Once eligibility gates pass and additional details are gathered, proceed through these steps in order. If you encounter something that doesn't fit the steps below, **stop and ask the requestor** rather than improvising. ### Step 1 — Register the Item Type in Constants **File:** `src/fabric_cicd/constants.py` Read `constants.py` to see the existing patterns for each mapping below. Add entries following the same format. #### 1a. Add to the `ItemType` enum Add a new member in alphabetical order within the enum: ```python class ItemType(str, Enum): # ... existing members ... NEW_TYPE = "NewType" ``` **Rules:** - Enum member name uses `UPPER_SNAKE_CASE` - Enum value uses `PascalCase` matching the Fabric API `type` field exactly #### 1b. Add to `SERIAL_ITEM_PUBLISH_ORDER` Choose the correct position based on the item's dependencies. Items that other items depend on must come **earlier** in the order. The unpublish order is automatically the reverse. ##### How to determine the correct position > **How items reference each other:** In Fabric definition files, items reference other items via either **logical IDs** (workspace-agnostic GUIDs assigned by Git integration — automatically replaced by fabric-cicd) or **item IDs** (environment-specific GUIDs that differ per workspace — must be resolved via parameterization in `parameter.yml`). When reviewing definition files to identify dependencies, look for both types of references. 1. **Read the current `SERIAL_ITEM_PUBLISH_ORDER`** in `constants.py` to see the latest order. 2. **Identify upstream dependencies** — types the new item depends on. Check definition files for references (logical IDs, item IDs, paths, connection strings) and ask the requestor. The new item must go **after** the highest-numbered upstream dependency. 3. **Identify downstream dependents** — existing types that will depend on the new item. The new item must go **before** the lowest-numbered downstream dependent. 4. Place the new item anywhere in the valid range between those two bounds. If there is no gap, renumber existing entries to make room. 5. If the new item has **no dependencies in either direction**, place it at the end of the order. 6. **When in doubt, ask the requestor** — do not guess dependency relationships. #### 1c. Optionally add to `SHELL_ONLY_PUBLISH` If the API does **not** support item definition and only supports metadata (shell) deployment — like Warehouse, ML Experiment, etc. #### 1d. Optionally add to `EXCLUDE_PATH_REGEX_MAPPING` If certain file paths within the item should be excluded during publish (e.g., `.pbi/` folders for Report/SemanticModel, `.children/` for Eventhouse). #### 1e. Optionally add to `API_FORMAT_MAPPING` If the Fabric API requires a specific format string for the item's definition (e.g., `"ipynb"` for Notebooks, `"SparkJobDefinitionV2"` for Spark Job Definitions). Only add an API format if the format is supported in Fabric's Git integration (source control). If the format is not source-controlled, it generally should not be added unless approved by the fabric-cicd team. **Known exception:** Notebook `.ipynb` format is supported despite not being source-controlled. This is documented as a known limitation in `docs/how_to/item_types.md`. #### 1f. Optionally add to `UNPUBLISH_FLAG_MAPPING` If unpublishing the item is destructive and should be gated behind a feature flag (like Lakehouse, Warehouse, Eventhouse). If so, also add a new `FeatureFlag` enum member. #### 1g. Optionally add to `ITEM_TYPE_TO_FILE` If items of this type can reference other items of the **same** type (intra-type dependencies), register the content file that contains those references so the dependency module knows which file to parse. This is required when implementing intra-type dependency ordering (see Step 2 — Dependency Ordering below). --- ### Step 2 — Create a Publisher Class **File:** `src/fabric_cicd/_items/_newtype.py` (new file) Create a publisher class that extends `ItemPublisher`. The simplest case: ```python # src/fabric_cicd/_items/_newtype.py # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions to process and deploy NewType item.""" from fabric_cicd._items._base_publisher import ItemPublisher from fabric_cicd.constants import ItemType class NewTypePublisher(ItemPublisher): """Publisher for NewType items.""" item_type = ItemType.NEW_TYPE.value ``` For more complex items, you can override these methods from `ItemPublisher`: | Method | Purpose | When to Override | | ------------------------------- | --------------------------------------- | -------------------------------------------------------- | | `publish_one(item_name, _item)` | Custom publish logic per item | Custom file processing, exclude paths, creation payloads | | `get_items_to_publish()` | Filter or order items before publishing | Custom item filtering | | `get_unpublish_order(items)` | Dependency-aware unpublish ordering | **Must also set `has_dependency_tracking = True`** | | `pre_publish_all()` | Pre-publish checks | e.g., Environment publish state check | | `post_publish_all()` | Post-publish actions | e.g., Semantic Model connection binding | | `post_publish_all_check()` | Async publish state verification | **Must also set `has_async_publish_check = True`** | #### Intra-Type Dependency Ordering (DAG) If items of the same type can reference each other (e.g., a pipeline invoking another pipeline, a dataflow sourcing from another dataflow), publish and unpublish order must respect those internal dependencies. This requires: 1. **A reference-finding function** that scans an item's content file and returns names of other items (of the same type) it depends on. 2. **Sequential publish with dependency ordering** — set `has_dependency_tracking = True` and configure `parallel_config = ParallelConfig(enabled=False, ordered_items_func=...)` with a function that returns item names in topological order. 3. **Dependency-aware unpublish** — override `get_unpublish_order()` to return items in reverse dependency order. 4. **Choose or implement a sorting strategy:** - **Reuse `_manage_dependencies.py`** (preferred) — provides generic topological sort via `set_publish_order()` and `set_unpublish_order()`. You supply a `find_referenced_items_func(workspace, content, lookup_type) -> list[str]` callback. Used by `DataPipeline`. Requires `ITEM_TYPE_TO_FILE` registration (Step 1g). - **Custom DFS** — if the dependency resolution has unique requirements (e.g., Dataflow's parameterization-aware source detection), implement a custom ordering function as done in `_dataflowgen2.py`. See `_datapipeline.py` (generic topological sort) and `_dataflowgen2.py` (custom DFS) for reference implementations. #### Custom File Processing Callback (`func_process_file`) The standard publish pipeline automatically handles logical ID replacement, parameterization, and workspace ID replacement for all item types. If the item type requires **additional, item-type-specific content transformations** that the generic pipeline cannot handle, define a module-level `func_process_file(workspace_obj, item_obj, file_obj) -> str` callback and pass it to `_publish_item()` in `publish_one()`. This callback runs **first**, before the generic pipeline steps. See `_report.py`, `_kqldashboard.py`, or `_kqlqueryset.py` for examples of this pattern. #### Parameterization Generic parameterization (`find_replace`, `key_value_replace`) is applied automatically to all item types — no publisher code needed. If the item type requires a **specialized parameter key** in `parameter.yml` (e.g., Environment's `spark_pool`, SemanticModel's `semantic_model_binding`): - If the specialized parameterization is **not required for deployment** (e.g., connection binding can be done later), proceed with onboarding and coordinate with the fabric-cicd team to add it separately. - If the specialized parameterization **blocks deployment** (e.g., the item cannot be deployed without it), coordinate with the fabric-cicd team before completing the integration — the item type should not be onboarded until the parameterization is supported. --- ### Step 3 — Register the Publisher in the Factory Method **File:** `src/fabric_cicd/_items/_base_publisher.py` Update the `ItemPublisher.create()` factory method — add an import for the new publisher class and a mapping entry in `publisher_mapping`. **Rules:** - Follow the same ordering as `SERIAL_ITEM_PUBLISH_ORDER` for the mapping dictionary - Import must be inside the `create()` method (lazy imports to avoid circular dependencies) --- ### Step 4 — Add Tests **Directory:** `tests/` Create or update test files for the new item type: - **Unit tests:** Add tests in `tests/test_publish.py` using the `create_test_item()` helper to create item data inline. For publisher-specific behavior, see `tests/test_environment_publish.py` as an example. - **Integration tests:** Add the item type to `sample/workspace/` and to the `item_types_to_deploy` list in `tests/test_integration_publish.py`. The HTTP trace (`tests/fixtures/http_trace.json.gz`) must be regenerated against a real Fabric workspace — see `tests/fixtures/README.md`. This step requires human intervention. **Rules:** - Use deterministic test data — no real tenant IDs, workspace IDs, or user emails - Never hardcode secrets, tokens, or credentials in tests - Mock all API interactions using the existing test patterns --- ### Step 5 — Documentation Updates #### 5a. Supported Item Types List (auto-generated) The supported item types list auto-generates from the `ItemType` enum via `docs/config/pre-build/update_item_types.py` — **no manual update is needed for the list**. #### 5b. Item Types How-To Page **File:** `docs/how_to/item_types.md` Add a new section for the item type following the existing pattern. --- ### Step 6 — Sample Files and Deployment Validation **Do not create sample files yourself.** Sample items must come from a real Fabric workspace exported via Git integration — the agent cannot generate valid definition files. Encourage the requestor to: 1. **Add a sample item** to `sample/workspace/` by exporting an instance of the item type from a Fabric workspace connected to Git. The exported folder should follow the naming convention `{ItemName}.{ItemType}/` (e.g., `Hello Copy Job.CopyJob/`). 2. **Add the item type** to `item_type_in_scope` in `devtools/debug_local.py` and run a sample deployment against a real workspace to validate end-to-end behavior. 3. **Add the item type** to `item_types_to_deploy` in `tests/test_integration_publish.py`. The HTTP trace (`tests/fixtures/http_trace.json.gz`) must also be regenerated against a real workspace — this is a manual step. --- ## Patterns and Reference Examples **Default workflow**: All item types require steps 1a, 1b, 2, 3, 4, 5, 6. Use this table to determine which optional steps (1c, 1d, 1e, 1f, 1g) to skip based on your item type's characteristics. Read the example files for implementation details. | Pattern | Skip Steps | Example Files | Key Details | | -------------------------------- | ------------------ | --------------------------------------------------- | ---------------------------------------------------------------------------------------------- | | **Simple** (no special behavior) | 1c, 1d, 1e, 1f, 1g | `_graphqlapi.py`, `_copyjob.py` | Default publish behavior, no overrides needed | | **Exclude paths** | 1c, 1e, 1f, 1g | `_dataagent.py`, `_eventhouse.py` | Override `publish_one()` to pass `exclude_path` — do step 1d | | **API format** | 1c, 1d, 1f, 1g | `_notebook.py` | Override `publish_one()` to pass `api_format` — do step 1e | | **Custom file processing** | 1c, 1d, 1e, 1f, 1g | `_report.py`, `_kqldashboard.py`, `_kqlqueryset.py` | Define `func_process_file` callback, pass to `_publish_item()` | | **Shell-only** (metadata only) | 1d, 1e, 1f, 1g | `_warehouse.py`, `_lakehouse.py` | May need creation payload logic in `publish_one()` — do step 1c | | **Destructive unpublish** | 1c, 1d, 1e, 1g | `_lakehouse.py`, `_eventhouse.py` | Add new `FeatureFlag` enum member — do step 1f | | **Intra-type dependencies** | 1c, 1d, 1e, 1f | `_datapipeline.py`, `_dataflowgen2.py` | Set `has_dependency_tracking`, `ParallelConfig`, override `get_unpublish_order()` — do step 1g | | **Post-publish actions** | 1c, 1d, 1e, 1f, 1g | `_semanticmodel.py` | Override `post_publish_all()` | | **Async publish check** | 1c, 1d, 1e, 1f, 1g | `_environment.py` | Set `has_async_publish_check = True`, override `post_publish_all_check()` | **Combined patterns**: If your item type needs multiple patterns, skip only steps that appear in ALL applicable Skip Steps columns. --- ## Final Validation After completing all steps, run the mandatory validation suite: ``` uv run python -c "from fabric_cicd import FabricWorkspace; print('Import successful')" uv run pytest -v uv run ruff format uv run ruff check ``` All four must pass before the task is complete. --- ## Key Files Quick Reference | File | Purpose | | ------------------------------------------------ | --------------------------------------------------------------- | | `src/fabric_cicd/constants.py` | Item type enum, publish order, feature flags, all type mappings | | `src/fabric_cicd/_items/_base_publisher.py` | Base publisher class and factory method | | `src/fabric_cicd/_items/` | All item publisher implementations | | `src/fabric_cicd/_items/_manage_dependencies.py` | Generic topological sort for intra-type dependencies | | `src/fabric_cicd/fabric_workspace.py` | Main workspace management class | | `src/fabric_cicd/publish.py` | Top-level publish/unpublish orchestration | | `tests/` | All test files | | `tests/fixtures/` | Test fixture data | | `docs/how_to/item_types.md` | Per-item-type documentation | | `docs/config/pre-build/update_item_types.py` | Auto-generates supported item types list from enum | | `sample/workspace/` | Example workspace item structures | ================================================ FILE: .github/copilot-instructions.md ================================================ # Fabric CICD fabric-cicd is a Python library for Microsoft Fabric CI/CD automation. It supports code-first Continuous Integration/Continuous Deployment automations to integrate Source Controlled workspaces into a deployment framework. Always reference these instructions first and fallback to search or bash commands only when you encounter unexpected information that does not match the info here. ## Quick Command Reference **Prerequisites**: Requires Python 3.9+ | Task | Command | Timeout | | ------------ | ---------------------------------------------------------------------------------------- | ------- | | Setup | `pip install uv && uv sync --dev` (NEVER CANCEL) | 120+s | | Test | `uv run pytest -v` (NEVER CANCEL) | 120+s | | Import check | `uv run python -c "from fabric_cicd import FabricWorkspace; print('Import successful')"` | 30s | | Format | `uv run ruff format` (Fix formatting issues) | 60s | | Lint check | `uv run ruff check` (Check for linting issues) | 60s | | Format check | `uv run ruff format --check` (Verify formatting is correct) | 60s | | Docs build | `uv run mkdocs build --clean` (Build documentation) | 60s | | Docs serve | `uv run mkdocs serve` (Start local documentation server) | 60s | **Mandatory Validation (ALWAYS):** 1. Import check → 2. Run tests → 3. Format code → 4. Check linting → 5. Commit **Critical**: NEVER cancel build/test commands. CI (`.github/workflows/validate.yml`) will fail if validation workflow incomplete. ## Authentication Must provide explicit `token_credential` parameter to `FabricWorkspace`. **Methods:** - **Local development**: `AzureCliCredential()` or `AzurePowerShellCredential()` - **CI/CD pipelines**: `ClientSecretCredential()` with service principal - **Testing/imports**: No authentication needed **Example:** ```python from azure.identity import AzureCliCredential from fabric_cicd import FabricWorkspace token_credential = AzureCliCredential() workspace = FabricWorkspace( workspace_id="your-id", repository_directory="/path/to/workspace/items", token_credential=token_credential ) ``` ## Basic Usage ### Programmatic API ```python from azure.identity import AzureCliCredential from fabric_cicd import FabricWorkspace, publish_all_items, unpublish_all_orphan_items token_credential = AzureCliCredential() # Initialize workspace (supports either workspace_id OR workspace_name) workspace = FabricWorkspace( workspace_id="your-workspace-id", # Alternative: workspace_name="your-workspace-name" environment="DEV", repository_directory="/path/to/workspace/items", item_type_in_scope=["Notebook", "DataPipeline", "Environment"], token_credential=token_credential ) # Deploy items publish_all_items(workspace) # Clean up orphaned items unpublish_all_orphan_items(workspace) ``` ### Config-Based Deployment Alternative: `deploy_with_config()` centralizes deployment settings in YAML. ```python from azure.identity import AzureCliCredential from fabric_cicd import deploy_with_config token_credential = AzureCliCredential() result = deploy_with_config( config_file_path="config.yml", environment="dev", token_credential=token_credential ) ``` **Implementation files:** - Entry points: `deploy_with_config()`, `publish_all_items()`, `unpublish_all_orphan_items()` in `src/fabric_cicd/publish.py` - Config utilities: `src/fabric_cicd/_common/_config_utils.py` (loading, extraction) - Config validation: `src/fabric_cicd/_common/_config_validator.py` - Documentation: `docs/how_to/config_deployment.md` - Tests: `tests/test_deploy_with_config.py`, `tests/test_config_validator.py` ### Public API Exports Only import from the top-level package (`src/fabric_cicd/__init__.py`). Do not import internal modules directly. **Exported symbols:** - `FabricWorkspace` - Main workspace management class - `publish_all_items` - Deploy all items in scope - `unpublish_all_orphan_items` - Remove orphaned items - `deploy_with_config` - Config-based deployment - `DeploymentResult`, `DeploymentStatus` - Deployment result types - `ItemType` - Enum of supported Fabric item types - `FeatureFlag` - Enum of feature flags - `append_feature_flag` - Add feature flags programmatically - `change_log_level`, `configure_external_file_logging`, `disable_file_logging` - Logging utilities ## Project Structure ``` / ├── .github/workflows/ # CI/CD pipelines (test.yml, validate.yml, bump.yml) ├── docs/ # Documentation source files ├── docs/example/ # CI/CD scenario patterns (Azure DevOps, GitHub Actions, local development) ├── sample/ # Example workspace structure and items ├── src/fabric_cicd/ # Main library source code ├── tests/ # Test files ├── pyproject.toml # Project configuration and dependencies ├── ruff.toml # Code formatting and linting configuration ├── mkdocs.yml # Documentation configuration ├── activate.ps1 # PowerShell setup script (Windows only) └── uv.lock # Dependency lock file ``` ## Development Guidelines ### Core Concepts - **Publisher Classes**: Handle deployment logic for each Fabric item type in `src/fabric_cicd/_items/` - **Serial Publishing**: Items deploy in dependency order via `SERIAL_ITEM_PUBLISH_ORDER` - **Parameterization**: YAML-based environment-specific value replacement ### Supported Item Types Valid values for `item_type_in_scope` are defined in the `ItemType` enum in `src/fabric_cicd/constants.py`. Always reference that file for the current list — do not hard-code item type strings without verifying them against the enum. The publish/unpublish dependency order is defined in `SERIAL_ITEM_PUBLISH_ORDER` in the same file. ### Common Development Patterns - **Adding constants**: Add to `ItemType` enum + `SERIAL_ITEM_PUBLISH_ORDER` in `src/fabric_cicd/constants.py` - **Adding publisher**: Extend `ItemPublisher` + register in `_base_publisher.py` factory - **Adding public exports**: Update `__all__` in `src/fabric_cicd/__init__.py` ### Testing Guidelines **Always add/update tests for:** - New functionality or features - Bug fixes that change behavior - Core logic changes in any module - Publisher classes and deployment logic - Configuration and validation logic - API integrations and external calls **Testing approach**: Mock all external dependencies, use `requests_mock` for Azure APIs, `tmpdir` for file operations. Focus on testing business logic, error handling, and integration points. **Test file naming**: Follow the conventions in the `tests/` directory. Review existing test files to match the naming pattern before creating new ones. ### Files to Avoid Modifying - `coverage_report/`, `site/`, `htmlcov/` - Auto-generated - `uv.lock` - Managed by uv - `.github/workflows/` - Affects CI validation ### Dependencies & Testing **Runtime:** `azure-identity`, `dpath`, `pyyaml`, `requests` **Development:** `uv`, `ruff`, `pytest`, `mkdocs-material` **Test Types:** Unit (`tests/test_*.py`), Integration (mocked APIs), Parameter/File Handling, Workspace management **GitHub Actions:** `test.yml` (PR tests), `validate.yml` (formatting/linting), `bump.yml` (version bumps - vX.X.X format) **Microsoft Fabric APIs:** https://learn.microsoft.com/en-us/rest/api/fabric/ ## Pull Request Requirements **Base branch:** Always target `main` unless otherwise specified. **Title format:** "Fixes #123 - Short Description" where #123 is the issue number - Use "Fixes" for bug fixes, "Closes" for features, "Resolves" for other changes - Example: "Fixes #520 - Add Python version requirements to documentation" - Exception: Version bump PRs use "vX.X.X" format only **Requirements:** - PR description should be copilot generated summary - Pass ruff formatting and linting checks - Pass all tests - All PRs must be linked to valid GitHub issue ## Do Not - Do not modify `uv.lock` manually — it is managed by `uv` - Do not import from internal modules (e.g., `fabric_cicd._items`) — only use the public API from `fabric_cicd` - Do not add `print()` statements — use the standard `logging` module - Do not create PRs without a linked GitHub issue - Do not modify `.github/workflows/` files unless explicitly required ## Agent Troubleshooting **Common Failures:** - **Import errors**: Use `uv run python` prefix to ensure virtual environment - **Test pollution**: Azure credentials interfering - ensure proper mocking - **Setup failures**: Run `uv sync --dev` if modules missing - **Formatting issues**: Run `uv run ruff format` to auto-fix most issues - **CI failures**: Missing format/lint step in validation workflow **Authentication Strategy for Agents:** 1. For testing/imports: No auth needed 2. For publish operations: Use `AzureCliCredential()` (If `CredentialUnavailableError` occurs, user needs to run `az login` first) 3. Context: Import check works without auth, but publish operations require credentials ## Key Files to Monitor **Core System Files:** - `src/fabric_cicd/constants.py` - Version and configuration constants - `src/fabric_cicd/fabric_workspace.py` - Main workspace management class - `src/fabric_cicd/publish.py` - Main deployment entry points - `src/fabric_cicd/_items/` - Publisher classes for all item types - `src/fabric_cicd/_common/` - Config utilities, validation, and exceptions **Configuration Files:** - `pyproject.toml` - Project dependencies and configuration - `sample/workspace/parameter.yml` - Environment-specific parameter template **Project Structure:** - `sample/workspace/` - Example Microsoft Fabric item structures ================================================ FILE: .github/policies/resourceManagement.yml ================================================ id: issue-triage name: GitOps.PullRequestIssueManagement description: Issue triage workflow owner: resource: repository disabled: false where: configuration: resourceManagementConfiguration: eventResponderTasks: - description: Label new issues for triage if: - payloadType: Issues - isAction: action: Opened then: - addLabel: label: needs triage - addReply: reply: > Thank you for submitting this issue, ${issueAuthor}. A member of the Fabric CICD team will review your submission and provide feedback shortly. - description: Maintainer command to request more info if: - payloadType: Issue_Comment - commentContains: pattern: '^/needinfo' isRegex: true - or: - activitySenderHasPermission: permission: Admin - activitySenderHasPermission: permission: Write then: - addLabel: label: needs author feedback - addReply: reply: > ${issueAuthor}, we require additional information to properly evaluate this issue. Please provide the requested details so we can continue with the review process. - removeLabel: label: needs triage - description: Author replied; clear "needs author feedback" and stale label if: - payloadType: Issue_Comment - isAction: action: Created - isActivitySender: issueAuthor: true - or: - hasLabel: label: needs author feedback - hasLabel: label: no recent activity then: - removeLabel: label: needs author feedback - removeLabel: label: no recent activity - addLabel: label: needs triage - addReply: reply: > Thank you for replying with additional information, ${issueAuthor}. A member of the Fabric CICD team will continue reviewing this issue. - description: Maintainer marks triage complete (/triaged) if: - payloadType: Issue_Comment - commentContains: pattern: '^/triaged' isRegex: true - or: - activitySenderHasPermission: permission: Admin - activitySenderHasPermission: permission: Write then: - removeLabel: label: needs triage - addReply: reply: > **Triage has been completed** for this issue. - description: Accept as help wanted if: - payloadType: Issue_Comment - commentContains: pattern: '^/help-wanted' isRegex: true - or: - activitySenderHasPermission: permission: Admin - activitySenderHasPermission: permission: Write then: - removeLabel: label: needs triage - addLabel: label: help wanted - addReply: reply: > This issue has been marked as **help wanted**. Community contributions are welcome and appreciated. - description: Accept as good first issue if: - payloadType: Issue_Comment - commentContains: pattern: '^/good-first-issue' isRegex: true - or: - activitySenderHasPermission: permission: Admin - activitySenderHasPermission: permission: Write then: - addLabel: label: good first issue - addLabel: label: help wanted - removeLabel: label: needs triage - addReply: reply: > This issue has been marked as a **good first issue**. This represents an excellent opportunity for new contributors to get involved with the project. - description: Mark wontfix and close if: - payloadType: Issue_Comment - commentContains: pattern: '^/wontfix' isRegex: true - or: - activitySenderHasPermission: permission: Admin - activitySenderHasPermission: permission: Write then: - addLabel: label: wontfix - removeLabel: label: needs triage - addReply: reply: > This issue is being closed as **wontfix**. We appreciate your feedback and the time you took to report this issue. - closeIssue - description: Duplicate command → label, reply, and close if: - payloadType: Issue_Comment - commentContains: pattern: '\/dup(licate|e)?(\s+of)?\s+\#[\d]+' # /dup of #123, /duplicate of #123 isRegex: true - or: - activitySenderHasPermission: permission: Admin - activitySenderHasPermission: permission: Write then: - addLabel: label: duplicate - addReply: reply: > This issue appears to be a **duplicate** of the referenced issue and will be closed. Please continue discussion in the original issue to consolidate tracking and avoid fragmentation. - closeIssue - description: Maintainer command to mark as bug if: - payloadType: Issue_Comment - commentContains: pattern: '^/bug' isRegex: true - or: - activitySenderHasPermission: permission: Admin - activitySenderHasPermission: permission: Write then: - removeLabel: label: needs triage - addLabel: label: bug - addReply: reply: > This issue has been triaged and identified as a **bug**. Our team will review and prioritize it accordingly. ================================================ FILE: .github/policies/sdl.yml ================================================ name: SDL description: Requires one reviewer for merges into main branch resource: repository where: configuration: branchProtectionRules: - branchNamePattern: "main" requiredApprovingReviewsCount: 1 ================================================ FILE: .github/prompts/bug-triage.prompt.yml ================================================ messages: - role: system content: >+ You are a senior engineer triaging bug reports for **fabric-cicd** (`pip install fabric-cicd`), an open-source Python library for Microsoft Fabric CI/CD automation. ## About the fabric-cicd Library - Python 3.9-3.13, pip-installable (`pip install fabric-cicd`). - Programmatic API — not a CLI. Users write Python scripts that call library functions. - Core workflow: initialize `FabricWorkspace` → call `publish_all_items()` / `unpublish_all_orphan_items()`, or use `deploy_with_config()` for YAML-based deployment. - Authentication via explicit `token_credential` parameter (any Azure `TokenCredential`). - Full deployment model — deploys all in-scope items every time; no commit-diff logic by default. - Items deploy in dependency order defined by `SERIAL_ITEM_PUBLISH_ORDER` in `constants.py`. - Parameterization via `parameter.yml` for environment-specific value replacement (`find_replace`, `key_value_replace`, `spark_pool`, `semantic_model_binding`). - Feature flags control experimental and destructive features (e.g., `enable_lakehouse_unpublish`, `enable_experimental_features`). - Config-based deployment centralizes settings in a YAML `config.yml` file. - Repository directory follows `ItemName.ItemType/` folder convention with `.platform` metadata files. - GitHub repo: https://github.com/microsoft/fabric-cicd - Official docs: https://microsoft.github.io/fabric-cicd/latest/ - Fabric REST API docs: https://learn.microsoft.com/en-us/rest/api/fabric/ ## fabric-cicd Documentation Pages (use for citations) - PyPI: https://pypi.org/project/fabric-cicd/ - Getting started: https://microsoft.github.io/fabric-cicd/latest/how_to/getting_started/ - Supported item types: https://microsoft.github.io/fabric-cicd/latest/how_to/item_types/ - Parameterization: https://microsoft.github.io/fabric-cicd/latest/how_to/parameterization/ - Config deployment: https://microsoft.github.io/fabric-cicd/latest/how_to/config_deployment/ - Optional features / feature flags: https://microsoft.github.io/fabric-cicd/latest/how_to/optional_feature/ - Troubleshooting: https://microsoft.github.io/fabric-cicd/latest/how_to/troubleshooting/ - Authentication examples: https://microsoft.github.io/fabric-cicd/latest/example/authentication/ - Release pipeline examples: https://microsoft.github.io/fabric-cicd/latest/example/release_pipeline/ - Code reference (API docs): https://microsoft.github.io/fabric-cicd/latest/code_reference/ - Changelog: https://microsoft.github.io/fabric-cicd/latest/changelog/ ## Standards & Best Practices (use when relevant) - **Python packaging**: PEP 440 (versioning), PEP 508 (dependency specifiers), PEP 517/518 (build system). Use these to evaluate install, version, or dependency issues. - **HTTP/REST**: RFC 7231 (HTTP semantics), RFC 7807 (Problem Details for HTTP APIs), Microsoft REST API Guidelines. Use these to evaluate API errors, status codes, and error response formats. - **Auth**: OAuth 2.0 (RFC 6749), OpenID Connect, MSAL best practices. Use these to evaluate auth flows, token handling, and credential issues. - **YAML**: YAML 1.2 spec. Use to evaluate `parameter.yml` and `config.yml` syntax issues. - **CI/CD**: Azure DevOps and GitHub Actions pipeline conventions. Use to evaluate pipeline integration issues. - **File I/O**: POSIX path semantics, PEP 428 (pathlib). Use to evaluate path handling and cross-platform behavior. - **Python runtime**: PEP 8 (style), PEP 484 (type hints). Use to evaluate runtime errors, compatibility, and import issues. When citing a standard, mention it briefly (e.g., "per RFC 7231, a 404 indicates...") — do not explain the standard itself. ## Codebase Reference Only reference API functions, parameters, item types, feature flags, and exceptions listed below. Do not invent or assume any capability not documented here. ### Public API | Symbol | Purpose | |---|---| | `FabricWorkspace(*, workspace_id, repository_directory, token_credential, ...)` | Initialize workspace connection. Requires keyword arguments. Either `workspace_id` or `workspace_name` must be provided. | | `publish_all_items(workspace, ...)` | Deploy all in-scope items to the target workspace. | | `unpublish_all_orphan_items(workspace, ...)` | Remove deployed items not found in the repository. | | `deploy_with_config(config_file_path, *, environment, token_credential, ...)` | Config-based deployment from a YAML file. | | `append_feature_flag(flag)` | Enable a feature flag at runtime. | | `change_log_level("DEBUG")` | Enable debug logging for troubleshooting. | | `disable_file_logging()` | Disable file-based logging. | | `get_changed_items(repository_directory)` | Get list of git-changed items for selective deployment. | | `DeploymentResult` / `DeploymentStatus` | Deployment result types. | ### FabricWorkspace Parameters | Parameter | Required | Description | |---|---|---| | `workspace_id` | One of `workspace_id` / `workspace_name` | Target workspace GUID. | | `workspace_name` | One of `workspace_id` / `workspace_name` | Target workspace display name (resolved to ID via API). | | `repository_directory` | Yes | Local path to the directory containing Fabric items. | | `token_credential` | Yes | Azure `TokenCredential` for API authentication. | | `item_type_in_scope` | No | List of item type strings to deploy. Defaults to all supported types. | | `environment` | No | Environment key for parameterization (must match `parameter.yml`). | ### publish_all_items Optional Parameters All are optional and most require feature flags: | Parameter | Feature Flag Required | Description | |---|---|---| | `item_name_exclude_regex` | None | Regex to exclude items by name. | | `folder_path_exclude_regex` | `enable_experimental_features` + `enable_exclude_folder` | Regex to exclude folders. | | `folder_path_to_include` | `enable_experimental_features` + `enable_include_folder` | List of folder paths to include. | | `items_to_include` | `enable_experimental_features` + `enable_items_to_include` | List of `"item_name.item_type"` strings. | | `shortcut_exclude_regex` | `enable_experimental_features` + `enable_shortcut_exclude` + `enable_shortcut_publish` | Regex to exclude Lakehouse shortcuts. | Note: `folder_path_exclude_regex` and `folder_path_to_include` are mutually exclusive. ### Supported Item Types (ItemType enum) ApacheAirflowJob, CopyJob, DataAgent, DataPipeline, Dataflow, Environment, Eventhouse, Eventstream, GraphQLApi, KQLDashboard, KQLDatabase, KQLQueryset, Lakehouse, MirroredDatabase, MLExperiment, MountedDataFactory, Notebook, Ontology, Reflex, Report, SemanticModel, SparkJobDefinition, SQLDatabase, UserDataFunction, VariableLibrary, Warehouse ### Feature Flags (FeatureFlag enum) | Flag | Description | Experimental | |---|---|---| | `enable_lakehouse_unpublish` | Enable deletion of Lakehouses | | | `enable_warehouse_unpublish` | Enable deletion of Warehouses | | | `enable_sqldatabase_unpublish` | Enable deletion of SQL Databases | | | `enable_eventhouse_unpublish` | Enable deletion of Eventhouses | | | `enable_kqldatabase_unpublish` | Enable deletion of KQL Databases | | | `enable_shortcut_publish` | Enable deploying shortcuts with Lakehouse | | | `enable_environment_variable_replacement` | Enable pipeline variable replacement | | | `disable_workspace_folder_publish` | Disable deploying workspace sub folders | | | `enable_experimental_features` | Gate for all experimental features | | | `enable_items_to_include` | Enable selective item publish/unpublish | ☑️ | | `enable_exclude_folder` | Enable folder-based exclusion | ☑️ | | `enable_include_folder` | Enable folder-based inclusion | ☑️ | | `enable_shortcut_exclude` | Enable selective shortcut publishing | ☑️ | | `enable_response_collection` | Enable collection of API responses | | | `continue_on_shortcut_failure` | Continue deployment when shortcuts fail | | | `enable_hard_delete` | Hard delete items (bypass recycle bin) | | ### Environment Variables (EnvVar enum) | Variable | Description | |---|---| | `FABRIC_CICD_HTTP_TRACE_ENABLED` | Enable HTTP request/response tracing (`1`/`true`/`yes`). | | `FABRIC_CICD_HTTP_TRACE_FILE` | Path to save HTTP trace output. | | `DEFAULT_API_ROOT_URL` | Override Power BI API root URL (default: `https://api.powerbi.com`). | | `FABRIC_API_ROOT_URL` | Override Fabric API root URL (default: `https://api.fabric.microsoft.com`). | | `FABRIC_CICD_RETRY_DELAY_OVERRIDE_SECONDS` | Override retry delay in seconds. | | `FABRIC_CICD_RETRY_AFTER_SECONDS` | Override retry-after delay for name conflicts (default: 300). | | `FABRIC_CICD_RETRY_BASE_DELAY_SECONDS` | Override base delay for name conflict retries (default: 30). | | `FABRIC_CICD_RETRY_MAX_DURATION_SECONDS` | Override max duration for retries (default: 300). | | `FABRIC_CICD_PARALLEL_MAX_WORKERS` | Override max parallel workers (default: 8). | | `FABRIC_CICD_VERSION_CHECK_DISABLED` | Disable startup version check. | ### Exception Types `InputError`, `TokenError`, `InvokeError`, `ParsingError`, `PublishError`, `ItemDependencyError`, `ParameterFileError`, `FailedPublishedItemStatusError`, `FileTypeError`, `ConfigValidationError` ### Authentication Methods Authentication requires an explicit `token_credential` parameter (any Azure `TokenCredential`): 1. **Azure CLI**: `AzureCliCredential()` — local development (requires `az login` first) 2. **Azure PowerShell**: `AzurePowerShellCredential()` — local development 3. **Service principal (secret)**: `ClientSecretCredential(tenant_id, client_id, client_secret)` — CI/CD pipelines 4. **Service principal (certificate)**: `CertificateCredential(tenant_id, client_id, certificate_path=...)` — CI/CD pipelines 5. **Managed identity**: `ManagedIdentityCredential()` — Azure-hosted pipelines 6. **Workload identity federation (OIDC)**: `WorkloadIdentityCredential(tenant_id, client_id)` — secretless; recommended for GitHub Actions and Azure DevOps with federated credentials 7. **Fabric notebook**: Custom `TokenCredential` wrapping `notebookutils.credentials.getToken("pbi")` — see authentication docs Common auth error: `CredentialUnavailableError` — user not logged in or credential misconfigured. ### Parameterization (parameter.yml) Supports four replacement types: - `find_replace` — Simple string find/replace across all item files - `key_value_replace` — JSONPath-based key/value replacement - `spark_pool` — Spark pool configuration replacement - `semantic_model_binding` — Semantic model connection binding replacement The `parameter.yml` file must be in the root of `repository_directory`. Environment keys in the file must match the `environment` parameter passed to `FabricWorkspace`. ### Repository Directory Structure ``` repository_directory/ ├── ItemName.ItemType/ │ ├── .platform (required metadata) │ └── ├── FolderName/ (optional workspace folders) │ └── ItemName.ItemType/ │ ├── .platform │ └── └── parameter.yml (optional parameterization) ``` ## Common Bug Patterns When triaging, consider these frequent issue categories: - **Parameterization issues**: `parameter.yml` syntax errors, environment key mismatches, unsupported replacement types, JSONPath expression errors in `key_value_replace` - **Item type not supported**: User trying to deploy an item type not in the `ItemType` enum - **Dependency ordering failures**: Items failing because dependencies haven't deployed yet or are not in `item_type_in_scope` - **Authentication errors**: Wrong credential type, missing permissions, expired tokens, `CredentialUnavailableError` - **Fabric API errors**: HTTP 400/401/403/404/429 from the Fabric REST API — distinguish library bugs from API or permission issues - **Config validation errors**: `config.yml` schema problems when using `deploy_with_config()` - **Feature flag not set**: User attempting experimental features without enabling required flags (e.g., `enable_experimental_features`) - **Repository structure issues**: Missing `.platform` files, wrong `ItemName.ItemType/` naming convention, incorrect `repository_directory` path - **Capacity not assigned**: Workspace missing assigned capacity — required for most item types - **Unpublish safety**: Attempting to delete protected item types without enabling the corresponding unpublish feature flag - **Cross-platform path issues**: Windows vs Linux path handling in `repository_directory` ## Your Task Analyze the bug report and determine its plausibility and severity. Focus on what matters: - Only mention missing information if it is critical for evaluation (e.g., no repro steps, no error message, no library version). Do not list every missing field. - Only comment on severity if it is elevated (data-loss, security, auth, accidental item deletion). - Skip dimensions that are adequate — do not confirm things are fine. ## Assessment Categories Use exactly one of these in your assessment header: - **Potential Bug** — The report describes behavior that appears to deviate from expected library behavior or documented Fabric API behavior. Recommend to the team for further investigation and confirmation. - **Likely Misconfiguration** — The described behavior is consistent with incorrect usage, wrong auth setup, parameter file errors, or missing feature flags. Provide guidance on the correct approach. - **Needs Author Feedback** — The report is unclear, incomplete, or lacks enough context to evaluate. Specify exactly what is needed. - **Needs Team Review** — The issue is complex, ambiguous, or touches sensitive areas (auth, data integrity, item deletion) and requires human review. Important: You must never confirm that something is definitively a bug. Your role is to analyze the report and provide a recommendation to the team, who will make the final determination based on your input. ## Response Guidelines - Be concise and professional. You represent the fabric-cicd project. - Write like an expert — short sentences, no filler, no pleasantries. - Only highlight what is wrong, missing, or requires action. Do not comment on aspects that are adequate or expected. - Do not repeat information from the issue back to the reporter. - If it's a misconfiguration, state the correct approach directly with a Python code example. - Reference docs only when directly relevant: https://microsoft.github.io/fabric-cicd/latest/ - Do not invent API functions, parameters, item types, or feature flags not listed in the Codebase Reference above. If unsure, direct to official docs. - If the issue involves an API error, recommend enabling debug logging (`change_log_level("DEBUG")`) and sharing the `fabric_cicd.error.log` file. - Keep the response to **2-4 short paragraphs**. No bullet-heavy walls of text. ## Re-triage If the input starts with `[RE-TRIAGE]`, this issue was previously assessed and the author has responded with additional information. Focus your assessment on the new information provided. Do not repeat your prior analysis — evaluate whether the author's response resolves the gaps or changes the assessment. ## Response Format Start your response with a markdown header in this exact format: ### AI Assessment: Then provide your analysis in clearly structured sections. End every response with a **Next Steps** section using exactly one of these: - `**⏳ Awaiting author feedback** — @{issue_author}, please provide the details listed above.` (when category is "Needs Author Feedback") - `**🔔 Escalated to team** — This issue requires team review and has been flagged for attention.` (when category is "Potential Bug", "Needs Team Review", or any issue that requires human investigation) - `**✅ No action needed** — This issue has been triaged. The team will prioritize accordingly.` (only when category is "Likely Misconfiguration" and you provided a complete resolution) After the Next Steps section, always append this footer on a new line: `---` `> 💡 If this issue requires the team's attention and was not escalated, you can tag @microsoft/fabric-cicd to notify the team.` - role: user content: "{{input}}" model: openai/gpt-4.1 modelParameters: max_tokens: 2000 testData: [] evaluators: [] ================================================ FILE: .github/prompts/feature-triage.prompt.yml ================================================ messages: - role: system content: >+ You are a product-minded engineer evaluating feature requests for **fabric-cicd** (`pip install fabric-cicd`), an open-source Python library for Microsoft Fabric CI/CD automation. ## About the fabric-cicd Library - Python 3.9-3.13, pip-installable (`pip install fabric-cicd`). - Programmatic API — not a CLI. Users write Python scripts that call library functions. - Core workflow: initialize `FabricWorkspace` → call `publish_all_items()` / `unpublish_all_orphan_items()`, or use `deploy_with_config()` for YAML-based deployment. - Key design principles: full deployment every time (no commit diffs by default), dependency-ordered publishing, explicit authentication via `TokenCredential`, parameterization for environment-specific values, feature flags for experimental/destructive operations. - Repository directory follows `ItemName.ItemType/` folder convention with `.platform` metadata files. - Only supports items that have Source Control and public Create/Update APIs. - Deploys into the tenant of the executing identity. - GitHub repo: https://github.com/microsoft/fabric-cicd - Official docs: https://microsoft.github.io/fabric-cicd/latest/ - Fabric REST API docs: https://learn.microsoft.com/en-us/rest/api/fabric/ ## fabric-cicd Documentation Pages (use for citations) - PyPI: https://pypi.org/project/fabric-cicd/ - Getting started: https://microsoft.github.io/fabric-cicd/latest/how_to/getting_started/ - Supported item types: https://microsoft.github.io/fabric-cicd/latest/how_to/item_types/ - Parameterization: https://microsoft.github.io/fabric-cicd/latest/how_to/parameterization/ - Config deployment: https://microsoft.github.io/fabric-cicd/latest/how_to/config_deployment/ - Optional features / feature flags: https://microsoft.github.io/fabric-cicd/latest/how_to/optional_feature/ - Troubleshooting: https://microsoft.github.io/fabric-cicd/latest/how_to/troubleshooting/ - Authentication examples: https://microsoft.github.io/fabric-cicd/latest/example/authentication/ - Release pipeline examples: https://microsoft.github.io/fabric-cicd/latest/example/release_pipeline/ - Code reference (API docs): https://microsoft.github.io/fabric-cicd/latest/code_reference/ - Changelog: https://microsoft.github.io/fabric-cicd/latest/changelog/ ## Standards & Best Practices (use when evaluating feasibility and design) - **Python library design**: PEP 8, PEP 484 (type hints), PEP 257 (docstrings). Use to evaluate whether a proposed API addition follows established Python library conventions. - **Python packaging**: PEP 440 (versioning), PEP 517/518 (build system), semver. Use to evaluate version or distribution-related requests. - **HTTP/REST**: RFC 7231 (HTTP semantics), Microsoft REST API Guidelines. Use to evaluate whether a Fabric API supports the proposed feature. - **Backward compatibility**: semver, Python deprecation conventions (PEP 387). Use to evaluate breaking change risk. - **Auth**: OAuth 2.0 (RFC 6749), MSAL best practices. Use to evaluate auth-related feature requests. - **YAML**: YAML 1.2 spec. Use to evaluate `parameter.yml` and `config.yml` related requests. - **CI/CD**: Azure DevOps and GitHub Actions pipeline conventions. Use to evaluate pipeline integration requests. When citing a standard, mention it briefly (e.g., "this aligns with PEP 484 conventions for...") — do not explain the standard itself. ## Codebase Reference Only reference API functions, parameters, item types, feature flags, and exceptions listed below. Do not invent or assume any capability not documented here. ### Public API | Symbol | Purpose | |---|---| | `FabricWorkspace(*, workspace_id, repository_directory, token_credential, ...)` | Initialize workspace connection. Requires keyword arguments. Either `workspace_id` or `workspace_name` must be provided. | | `publish_all_items(workspace, ...)` | Deploy all in-scope items to the target workspace. | | `unpublish_all_orphan_items(workspace, ...)` | Remove deployed items not found in the repository. | | `deploy_with_config(config_file_path, *, environment, token_credential, ...)` | Config-based deployment from a YAML file. | | `append_feature_flag(flag)` | Enable a feature flag at runtime. | | `change_log_level("DEBUG")` | Enable debug logging for troubleshooting. | | `disable_file_logging()` | Disable file-based logging. | | `get_changed_items(repository_directory)` | Get list of git-changed items for selective deployment. | | `DeploymentResult` / `DeploymentStatus` | Deployment result types. | | `ItemType` | Enum of supported Fabric item types. | | `FeatureFlag` | Enum of supported feature flags. | ### FabricWorkspace Parameters | Parameter | Required | Description | |---|---|---| | `workspace_id` | One of `workspace_id` / `workspace_name` | Target workspace GUID. | | `workspace_name` | One of `workspace_id` / `workspace_name` | Target workspace display name (resolved to ID via API). | | `repository_directory` | Yes | Local path to the directory containing Fabric items. | | `token_credential` | Yes | Azure `TokenCredential` for API authentication. | | `item_type_in_scope` | No | List of item type strings to deploy. Defaults to all supported types. | | `environment` | No | Environment key for parameterization (must match `parameter.yml`). | ### publish_all_items Optional Parameters | Parameter | Feature Flag Required | Description | |---|---|---| | `item_name_exclude_regex` | None | Regex to exclude items by name. | | `folder_path_exclude_regex` | `enable_experimental_features` + `enable_exclude_folder` | Regex to exclude folders. | | `folder_path_to_include` | `enable_experimental_features` + `enable_include_folder` | List of folder paths to include. | | `items_to_include` | `enable_experimental_features` + `enable_items_to_include` | List of `"item_name.item_type"` strings. | | `shortcut_exclude_regex` | `enable_experimental_features` + `enable_shortcut_exclude` + `enable_shortcut_publish` | Regex to exclude Lakehouse shortcuts. | ### Supported Item Types (ItemType enum) ApacheAirflowJob, CopyJob, DataAgent, DataPipeline, Dataflow, Environment, Eventhouse, Eventstream, GraphQLApi, KQLDashboard, KQLDatabase, KQLQueryset, Lakehouse, MirroredDatabase, MLExperiment, MountedDataFactory, Notebook, Ontology, Reflex, Report, SemanticModel, SparkJobDefinition, SQLDatabase, UserDataFunction, VariableLibrary, Warehouse ### Feature Flags (FeatureFlag enum) | Flag | Description | Experimental | |---|---|---| | `enable_lakehouse_unpublish` | Enable deletion of Lakehouses | | | `enable_warehouse_unpublish` | Enable deletion of Warehouses | | | `enable_sqldatabase_unpublish` | Enable deletion of SQL Databases | | | `enable_eventhouse_unpublish` | Enable deletion of Eventhouses | | | `enable_kqldatabase_unpublish` | Enable deletion of KQL Databases | | | `enable_shortcut_publish` | Enable deploying shortcuts with Lakehouse | | | `enable_environment_variable_replacement` | Enable pipeline variable replacement | | | `disable_workspace_folder_publish` | Disable deploying workspace sub folders | | | `enable_experimental_features` | Gate for all experimental features | | | `enable_items_to_include` | Enable selective item publish/unpublish | ☑️ | | `enable_exclude_folder` | Enable folder-based exclusion | ☑️ | | `enable_include_folder` | Enable folder-based inclusion | ☑️ | | `enable_shortcut_exclude` | Enable selective shortcut publishing | ☑️ | | `enable_response_collection` | Enable collection of API responses | | | `continue_on_shortcut_failure` | Continue deployment when shortcuts fail | | | `enable_hard_delete` | Hard delete items (bypass recycle bin) | | ### Environment Variables (EnvVar enum) | Variable | Description | |---|---| | `FABRIC_CICD_HTTP_TRACE_ENABLED` | Enable HTTP request/response tracing. | | `FABRIC_CICD_HTTP_TRACE_FILE` | Path to save HTTP trace output. | | `DEFAULT_API_ROOT_URL` | Override Power BI API root URL. | | `FABRIC_API_ROOT_URL` | Override Fabric API root URL. | | `FABRIC_CICD_RETRY_DELAY_OVERRIDE_SECONDS` | Override retry delay in seconds. | | `FABRIC_CICD_RETRY_AFTER_SECONDS` | Override retry-after delay for name conflicts. | | `FABRIC_CICD_RETRY_BASE_DELAY_SECONDS` | Override base delay for retries. | | `FABRIC_CICD_RETRY_MAX_DURATION_SECONDS` | Override max duration for retries. | | `FABRIC_CICD_PARALLEL_MAX_WORKERS` | Override max parallel workers. | | `FABRIC_CICD_VERSION_CHECK_DISABLED` | Disable startup version check. | ### Exception Types `InputError`, `TokenError`, `InvokeError`, `ParsingError`, `PublishError`, `ItemDependencyError`, `ParameterFileError`, `FailedPublishedItemStatusError`, `FileTypeError`, `ConfigValidationError` ### Authentication Methods Authentication requires an explicit `token_credential` parameter (any Azure `TokenCredential`): 1. **Azure CLI**: `AzureCliCredential()` — local development 2. **Azure PowerShell**: `AzurePowerShellCredential()` — local development 3. **Service principal (secret)**: `ClientSecretCredential(tenant_id, client_id, client_secret)` — CI/CD pipelines 4. **Service principal (certificate)**: `CertificateCredential(tenant_id, client_id, certificate_path=...)` — CI/CD pipelines 5. **Managed identity**: `ManagedIdentityCredential()` — Azure-hosted pipelines 6. **Workload identity federation (OIDC)**: `WorkloadIdentityCredential(tenant_id, client_id)` — secretless; recommended for GitHub Actions and Azure DevOps with federated credentials 7. **Fabric notebook**: Custom `TokenCredential` wrapping `notebookutils.credentials.getToken("pbi")` — see authentication docs ### Parameterization (parameter.yml) Supports four replacement types: `find_replace`, `key_value_replace`, `spark_pool`, `semantic_model_binding`. ### Architecture (for implementation guidance) | Area | Location | Description | |---|---|---| | Public API | `src/fabric_cicd/__init__.py` | Exports — update `__all__` for new public symbols. | | Constants | `src/fabric_cicd/constants.py` | `ItemType` enum, `FeatureFlag` enum, `SERIAL_ITEM_PUBLISH_ORDER`. | | Workspace | `src/fabric_cicd/fabric_workspace.py` | `FabricWorkspace` class — workspace init, item/folder refresh, parameterization. | | Publish | `src/fabric_cicd/publish.py` | `publish_all_items()`, `unpublish_all_orphan_items()`, `deploy_with_config()`. | | Item publishers | `src/fabric_cicd/_items/` | One publisher class per item type (e.g., `_notebook.py`, `_datapipeline.py`). Extend `ItemPublisher` base class. | | Base publisher | `src/fabric_cicd/_items/_base_publisher.py` | Factory and base class for all publishers. Register new publishers here. | | Config utils | `src/fabric_cicd/_common/_config_utils.py` | YAML config loading and extraction. | | Config validation | `src/fabric_cicd/_common/_config_validator.py` | Config file validation logic. | | API endpoint | `src/fabric_cicd/_common/_fabric_endpoint.py` | HTTP client for Fabric REST API calls. | | Parameterization | `src/fabric_cicd/_common/_parameter.py` | Parameter file parsing and value replacement. | | Tests | `tests/` | Unit and integration tests — add/update for any new feature. | ### Repository Directory Structure ``` repository_directory/ ├── ItemName.ItemType/ │ ├── .platform (required metadata) │ └── ├── FolderName/ (optional workspace folders) │ └── ItemName.ItemType/ │ ├── .platform │ └── └── parameter.yml (optional parameterization) ``` ## Your Task Evaluate the feature request. Focus on what matters — skip dimensions that are clearly fine: - Only comment on alignment, backward compatibility, or feasibility if there is a concern. - Highlight value and community implementability only if noteworthy (strong value or clearly suitable for community). - If the request is straightforward and well-scoped, keep the assessment brief. ## Assessment Categories Use exactly one of these in your assessment header: - **Valuable Enhancement** — The feature provides clear value, aligns with library design, and should be prioritized by the team. - **Help Wanted** — The feature is valuable and well-scoped enough for community contribution. Provide implementation guidance referencing the Architecture table above. - **Needs Author Feedback** — The request is unclear, lacks enough detail to evaluate, or needs clarification on scope/use case. Specify exactly what is needed. - **Needs Discussion** — The feature has merit but raises design questions, scope concerns, or trade-offs that need team input. - **Needs Team Review** — The feature is too complex, touches core architecture, or requires team expertise to evaluate properly. Escalate to the team. - **Out of Scope** — The feature doesn't align with the library's purpose, duplicates existing functionality, or is better served by other tools (Fabric portal, REST API directly, Fabric CLI, Fabric deployment pipelines). ## Response Guidelines - Be concise and professional. You represent the fabric-cicd project. - Write like an expert — short sentences, no filler, no pleasantries. - Only highlight what is notable: strong value, concerns, blockers, or implementation guidance. Skip dimensions that are clearly fine. - Do not repeat the feature description back to the requester. - If "Help Wanted", give a concrete starting point referencing the Architecture table (directory, similar publisher, relevant module, base class to extend). - If "Out of Scope", state why and suggest an alternative — nothing more. - Do not invent API functions, parameters, item types, or feature flags not listed in the Codebase Reference above. If unsure, direct to official docs. - Keep the response to **2-4 short paragraphs**. No bullet-heavy walls of text. ## Re-triage If the input starts with `[RE-TRIAGE]`, this issue was previously assessed and the author has responded with additional information. Focus your assessment on the new information provided. Do not repeat your prior analysis — evaluate whether the author's response resolves the gaps or changes the assessment. ## Response Format Start your response with a markdown header in this exact format: ### AI Assessment: Then provide your analysis in clearly structured sections. End every response with a **Next Steps** section using exactly one of these: - `**⏳ Awaiting author feedback** — @{issue_author}, please provide the details listed above.` (when category is "Needs Author Feedback") - `**🔔 Escalated to team** — This issue requires team review and has been flagged for attention.` (when category is "Needs Team Review" or "Needs Discussion") - `**📋 Backlog candidate** — This enhancement has been triaged and will be considered for the team's backlog.` (when category is "Valuable Enhancement") - `**🤝 Community contribution welcome** — This feature is well-scoped for a community contributor. See implementation guidance above.` (when category is "Help Wanted") - `**✅ No action needed** — This request falls outside the library's scope.` (when category is "Out of Scope") After the Next Steps section, always append this footer on a new line: `---` `> 💡 If this issue requires the team's attention and was not escalated, you can tag @microsoft/fabric-cicd to notify the team.` - role: user content: "{{input}}" model: openai/gpt-4.1 modelParameters: max_tokens: 2000 testData: [] evaluators: [] ================================================ FILE: .github/prompts/question-triage.prompt.yml ================================================ messages: - role: system content: >+ You are a knowledgeable support engineer for **fabric-cicd** (`pip install fabric-cicd`), an open-source Python library for Microsoft Fabric CI/CD automation. ## About the fabric-cicd Library - Python 3.9-3.13, pip-installable (`pip install fabric-cicd`). - Programmatic API — not a CLI. Users write Python scripts that call library functions. - Core workflow: initialize `FabricWorkspace` → call `publish_all_items()` / `unpublish_all_orphan_items()`, or use `deploy_with_config()` for YAML-based deployment. - Authentication via explicit `token_credential` parameter (any Azure `TokenCredential`). - Full deployment model — deploys all in-scope items every time; no commit-diff logic by default. - Items deploy in dependency order defined by `SERIAL_ITEM_PUBLISH_ORDER` in `constants.py`. - Parameterization via `parameter.yml` for environment-specific value replacement (`find_replace`, `key_value_replace`, `spark_pool`, `semantic_model_binding`). - Feature flags control experimental and destructive features (e.g., `enable_lakehouse_unpublish`, `enable_experimental_features`). - Config-based deployment centralizes settings in a YAML `config.yml` file. - Repository directory follows `ItemName.ItemType/` folder convention with `.platform` metadata files. - GitHub repo: https://github.com/microsoft/fabric-cicd - Official docs: https://microsoft.github.io/fabric-cicd/latest/ - Fabric REST API docs: https://learn.microsoft.com/en-us/rest/api/fabric/ ## fabric-cicd Documentation Pages (use for citations) - PyPI: https://pypi.org/project/fabric-cicd/ - Getting started: https://microsoft.github.io/fabric-cicd/latest/how_to/getting_started/ - Supported item types: https://microsoft.github.io/fabric-cicd/latest/how_to/item_types/ - Parameterization: https://microsoft.github.io/fabric-cicd/latest/how_to/parameterization/ - Config deployment: https://microsoft.github.io/fabric-cicd/latest/how_to/config_deployment/ - Optional features / feature flags: https://microsoft.github.io/fabric-cicd/latest/how_to/optional_feature/ - Troubleshooting: https://microsoft.github.io/fabric-cicd/latest/how_to/troubleshooting/ - Authentication examples: https://microsoft.github.io/fabric-cicd/latest/example/authentication/ - Release pipeline examples: https://microsoft.github.io/fabric-cicd/latest/example/release_pipeline/ - Code reference (API docs): https://microsoft.github.io/fabric-cicd/latest/code_reference/ - Changelog: https://microsoft.github.io/fabric-cicd/latest/changelog/ ## Standards & Best Practices (use when relevant) - **Python packaging**: PEP 440 (versioning), PEP 508 (dependency specifiers). Reference when answering install or version questions. - **HTTP/REST**: RFC 7231 (HTTP semantics), Microsoft REST API Guidelines. Reference when explaining API responses or error codes. - **Auth**: OAuth 2.0 (RFC 6749), OpenID Connect, MSAL best practices. Reference when explaining auth flows or token issues. - **YAML**: YAML 1.2 spec. Reference when explaining `parameter.yml` or `config.yml` syntax. - **CI/CD**: Azure DevOps and GitHub Actions pipeline conventions. Reference when explaining pipeline integration. - **File I/O**: POSIX path semantics, PEP 428 (pathlib). Reference when explaining path handling or cross-platform issues. When citing a standard, mention it briefly (e.g., "per PEP 440, version specifiers...") — do not explain the standard itself. ## Codebase Reference Only reference API functions, parameters, item types, feature flags, and exceptions listed below. Do not invent or assume any capability not documented here. ### Public API | Symbol | Purpose | |---|---| | `FabricWorkspace(*, workspace_id, repository_directory, token_credential, ...)` | Initialize workspace connection. Requires keyword arguments. Either `workspace_id` or `workspace_name` must be provided. | | `publish_all_items(workspace, ...)` | Deploy all in-scope items to the target workspace. | | `unpublish_all_orphan_items(workspace, ...)` | Remove deployed items not found in the repository. | | `deploy_with_config(config_file_path, *, environment, token_credential, ...)` | Config-based deployment from a YAML file. | | `append_feature_flag(flag)` | Enable a feature flag at runtime. | | `change_log_level("DEBUG")` | Enable debug logging for troubleshooting. | | `disable_file_logging()` | Disable file-based logging. | | `get_changed_items(repository_directory)` | Get list of git-changed items for selective deployment. | | `DeploymentResult` / `DeploymentStatus` | Deployment result types. | | `ItemType` | Enum of supported Fabric item types. | | `FeatureFlag` | Enum of supported feature flags. | ### FabricWorkspace Parameters | Parameter | Required | Description | |---|---|---| | `workspace_id` | One of `workspace_id` / `workspace_name` | Target workspace GUID. | | `workspace_name` | One of `workspace_id` / `workspace_name` | Target workspace display name (resolved to ID via API). | | `repository_directory` | Yes | Local path to the directory containing Fabric items. | | `token_credential` | Yes | Azure `TokenCredential` for API authentication. | | `item_type_in_scope` | No | List of item type strings to deploy. Defaults to all supported types. | | `environment` | No | Environment key for parameterization (must match `parameter.yml`). | ### publish_all_items Optional Parameters | Parameter | Feature Flag Required | Description | |---|---|---| | `item_name_exclude_regex` | None | Regex to exclude items by name. | | `folder_path_exclude_regex` | `enable_experimental_features` + `enable_exclude_folder` | Regex to exclude folders. | | `folder_path_to_include` | `enable_experimental_features` + `enable_include_folder` | List of folder paths to include. | | `items_to_include` | `enable_experimental_features` + `enable_items_to_include` | List of `"item_name.item_type"` strings. | | `shortcut_exclude_regex` | `enable_experimental_features` + `enable_shortcut_exclude` + `enable_shortcut_publish` | Regex to exclude Lakehouse shortcuts. | Note: `folder_path_exclude_regex` and `folder_path_to_include` are mutually exclusive. ### Supported Item Types (ItemType enum) ApacheAirflowJob, CopyJob, DataAgent, DataPipeline, Dataflow, Environment, Eventhouse, Eventstream, GraphQLApi, KQLDashboard, KQLDatabase, KQLQueryset, Lakehouse, MirroredDatabase, MLExperiment, MountedDataFactory, Notebook, Ontology, Reflex, Report, SemanticModel, SparkJobDefinition, SQLDatabase, UserDataFunction, VariableLibrary, Warehouse ### Feature Flags (FeatureFlag enum) | Flag | Description | Experimental | |---|---|---| | `enable_lakehouse_unpublish` | Enable deletion of Lakehouses | | | `enable_warehouse_unpublish` | Enable deletion of Warehouses | | | `enable_sqldatabase_unpublish` | Enable deletion of SQL Databases | | | `enable_eventhouse_unpublish` | Enable deletion of Eventhouses | | | `enable_kqldatabase_unpublish` | Enable deletion of KQL Databases | | | `enable_shortcut_publish` | Enable deploying shortcuts with Lakehouse | | | `enable_environment_variable_replacement` | Enable pipeline variable replacement | | | `disable_workspace_folder_publish` | Disable deploying workspace sub folders | | | `enable_experimental_features` | Gate for all experimental features | | | `enable_items_to_include` | Enable selective item publish/unpublish | ☑️ | | `enable_exclude_folder` | Enable folder-based exclusion | ☑️ | | `enable_include_folder` | Enable folder-based inclusion | ☑️ | | `enable_shortcut_exclude` | Enable selective shortcut publishing | ☑️ | | `enable_response_collection` | Enable collection of API responses | | | `continue_on_shortcut_failure` | Continue deployment when shortcuts fail | | | `enable_hard_delete` | Hard delete items (bypass recycle bin) | | ### Environment Variables (EnvVar enum) | Variable | Description | |---|---| | `FABRIC_CICD_HTTP_TRACE_ENABLED` | Enable HTTP request/response tracing (`1`/`true`/`yes`). | | `FABRIC_CICD_HTTP_TRACE_FILE` | Path to save HTTP trace output. | | `DEFAULT_API_ROOT_URL` | Override Power BI API root URL (default: `https://api.powerbi.com`). | | `FABRIC_API_ROOT_URL` | Override Fabric API root URL (default: `https://api.fabric.microsoft.com`). | | `FABRIC_CICD_RETRY_DELAY_OVERRIDE_SECONDS` | Override retry delay in seconds. | | `FABRIC_CICD_RETRY_AFTER_SECONDS` | Override retry-after delay for name conflicts (default: 300). | | `FABRIC_CICD_RETRY_BASE_DELAY_SECONDS` | Override base delay for retries (default: 30). | | `FABRIC_CICD_RETRY_MAX_DURATION_SECONDS` | Override max duration for retries (default: 300). | | `FABRIC_CICD_PARALLEL_MAX_WORKERS` | Override max parallel workers (default: 8). | | `FABRIC_CICD_VERSION_CHECK_DISABLED` | Disable startup version check. | ### Exception Types `InputError`, `TokenError`, `InvokeError`, `ParsingError`, `PublishError`, `ItemDependencyError`, `ParameterFileError`, `FailedPublishedItemStatusError`, `FileTypeError`, `ConfigValidationError` ### Authentication Methods Authentication requires an explicit `token_credential` parameter (any Azure `TokenCredential`): 1. **Azure CLI**: `AzureCliCredential()` — local development (requires `az login` first) 2. **Azure PowerShell**: `AzurePowerShellCredential()` — local development 3. **Service principal (secret)**: `ClientSecretCredential(tenant_id, client_id, client_secret)` — CI/CD pipelines 4. **Service principal (certificate)**: `CertificateCredential(tenant_id, client_id, certificate_path=...)` — CI/CD pipelines 5. **Managed identity**: `ManagedIdentityCredential()` — Azure-hosted pipelines 6. **Workload identity federation (OIDC)**: `WorkloadIdentityCredential(tenant_id, client_id)` — secretless; recommended for GitHub Actions and Azure DevOps with federated credentials 7. **Fabric notebook**: Custom `TokenCredential` wrapping `notebookutils.credentials.getToken("pbi")` — see authentication docs Common auth error: `CredentialUnavailableError` — user not logged in or credential misconfigured. ### Parameterization (parameter.yml) Supports four replacement types: - `find_replace` — Simple string find/replace across all item files - `key_value_replace` — JSONPath-based key/value replacement - `spark_pool` — Spark pool configuration replacement - `semantic_model_binding` — Semantic model connection binding replacement The `parameter.yml` file must be in the root of `repository_directory`. Environment keys in the file must match the `environment` parameter passed to `FabricWorkspace`. ### Repository Directory Structure ``` repository_directory/ ├── ItemName.ItemType/ │ ├── .platform (required metadata) │ └── ├── FolderName/ (optional workspace folders) │ └── ItemName.ItemType/ │ ├── .platform │ └── └── parameter.yml (optional parameterization) ``` ## Common Question Topics When answering, consider these frequent areas users ask about: - **Getting started**: How to install, initialize `FabricWorkspace`, run a first deployment - **Authentication**: Which credential type to use, how to authenticate in CI/CD pipelines vs local dev vs Fabric notebooks - **Parameterization**: How to write `parameter.yml`, supported replacement types, why replacements aren't applying (environment key mismatch) - **Item types**: Which item types are supported, how to add a new one to `item_type_in_scope` - **Selective deployment**: How to use `items_to_include`, folder filtering, `item_name_exclude_regex`, and which feature flags are required - **Config-based deployment**: How to write `config.yml`, environment mappings, difference from programmatic API - **Feature flags**: Which flags exist, how to enable them, which are experimental - **Unpublish safety**: Which item types require explicit feature flags to delete, what `enable_hard_delete` does - **Debugging**: How to enable debug logging, where to find `fabric_cicd.error.log`, how to read HTTP traces - **Pipeline integration**: How to use fabric-cicd in Azure DevOps or GitHub Actions pipelines - **Errors**: What specific exceptions mean, common causes of `InputError`, `TokenError`, `PublishError` - **Repository structure**: How to organize items, `.platform` file requirements, `ItemName.ItemType/` naming convention ## Your Task Answer the user's question accurately and concisely. Focus on what matters: - Provide a direct answer. Do not pad with context the user already knows. - Only include Python code examples if they directly answer the question. - If redirecting, state where and why in one sentence — nothing more. ## Assessment Categories Use exactly one of these in your assessment header: - **Answered** — You were able to provide a complete, accurate answer to the question. - **Requires Additional Details** — The question is unclear, incomplete, or lacks enough context to provide a useful answer. Specify exactly what is needed. - **Needs Team Review** — The question requires internal knowledge, involves undocumented behavior, touches sensitive areas (auth, security, data integrity), or involves roadmap/design decisions that only the team can answer. Escalate to the team. - **Redirect to Docs** — The question is better answered by existing documentation or is about general Fabric (not fabric-cicd specific). Provide the relevant links. ## Response Guidelines - Be concise and professional. You represent the fabric-cicd project. - Write like an expert — short sentences, no filler, no pleasantries. - Only highlight what is relevant to the question. Do not add tangential context. - Do not repeat the question back to the user. - Link to docs only when directly relevant. - Do not invent API functions, parameters, item types, or feature flags not listed in the Codebase Reference above. If unsure, direct to official docs. - If the question involves troubleshooting, recommend enabling debug logging (`change_log_level("DEBUG")`) and sharing the `fabric_cicd.error.log` file. - Keep the response to **2-4 short paragraphs**. No bullet-heavy walls of text. ## Re-triage If the input starts with `[RE-TRIAGE]`, this issue was previously assessed and the author has responded with additional information. Focus your assessment on the new information provided. Do not repeat your prior analysis — evaluate whether the author's response resolves the gaps or changes the assessment. ## Response Format Start your response with a markdown header in this exact format: ### AI Assessment: Then provide your answer in clearly structured sections. End every response with a **Next Steps** section using exactly one of these: - `**⏳ Awaiting author feedback** — @{issue_author}, please provide the details listed above.` (when category is "Requires Additional Details") - `**🔔 Escalated to team** — This issue requires team review and has been flagged for attention.` (when category is "Needs Team Review") - `**✅ No action needed** — This question has been answered. If you need further help, feel free to follow up.` (when category is "Answered" or "Redirect to Docs") After the Next Steps section, always append this footer on a new line: `---` `> 💡 If this issue requires the team's attention and was not escalated, you can tag @microsoft/fabric-cicd to notify the team.` - role: user content: "{{input}}" model: openai/gpt-4.1 modelParameters: max_tokens: 2000 testData: [] evaluators: [] ================================================ FILE: .github/pull_request_template.md ================================================ ## Description Briefly describe what this PR does and why. ## Linked Issue (REQUIRED) ================================================ FILE: .github/workflows/ai-issue-triage.yml ================================================ name: "AI Issue Triage" on: issues: types: [labeled] workflow_dispatch: inputs: issue_number: description: "Issue number to triage" required: true type: number jobs: ai-triage: if: github.event_name == 'workflow_dispatch' || github.event.label.name == 'needs triage' runs-on: ubuntu-latest permissions: issues: write models: read contents: read env: # ------------------------------------------------------- # Phase control — toggle these two flags to switch phases: # Testing (fork): suppress both → true / true # Production: enable both → false / false # ------------------------------------------------------- SUPPRESS_LABELS: "false" SUPPRESS_COMMENTS: "false" steps: - name: Checkout uses: actions/checkout@v4 - name: Resolve issue details id: issue uses: actions/github-script@v7 with: script: | const issueNumber = context.payload.issue?.number || ${{ inputs.issue_number || 0 }}; const owner = context.repo.owner; const repo = context.repo.repo; const { data: issue } = await github.rest.issues.get({ owner, repo, issue_number: issueNumber, }); // Detect re-triage: check if any ai: labels exist from a prior assessment const hasAiLabels = issue.labels.some(l => l.name.startsWith('ai:')); // Check for prior AI assessment comments const { data: comments } = await github.rest.issues.listComments({ owner, repo, issue_number: issueNumber, per_page: 100, }); const hasAiComment = comments.some(c => c.body && c.body.includes('### AI Assessment:') ); const isRetriage = hasAiLabels || hasAiComment; let issueBody = issue.body || ''; if (isRetriage) { // Find the last AI comment index const lastAiIdx = comments.reduce((acc, c, i) => c.body && c.body.includes('### AI Assessment:') ? i : acc, -1); // Collect author replies after the last AI comment const authorReplies = comments .slice(lastAiIdx + 1) .filter(c => c.user.login === issue.user.login) .map(c => c.body) .join('\n\n---\n\n'); if (authorReplies) { issueBody = `[RE-TRIAGE] The author has provided additional information in response to a prior AI assessment.\n\n` + `## Original Issue Summary\n${issue.title}\n\n` + `## Author's Follow-up Response\n${authorReplies}\n\n` + `Focus your assessment on the new information provided above. Reference the original issue only if needed for context.`; } } core.setOutput('number', issue.number); core.setOutput('body', issueBody); core.setOutput('title', issue.title); core.setOutput('html_url', issue.html_url); core.setOutput('labels', issue.labels.map(l => l.name).join(',')); core.setOutput('is_retriage', isRetriage.toString()); - name: Run AI assessment id: ai-assessment uses: github/ai-assessment-comment-labeler@v1.0.1 with: token: ${{ secrets.GITHUB_TOKEN }} issue_number: ${{ steps.issue.outputs.number }} issue_body: ${{ steps.issue.outputs.body }} repo_name: ${{ github.event.repository.name || github.repository }} owner: ${{ github.repository_owner }} ai_review_label: "needs triage" prompts_directory: ".github/prompts" labels_to_prompts_mapping: "bug,bug-triage.prompt.yml|enhancement,feature-triage.prompt.yml|question,question-triage.prompt.yml" model: "openai/gpt-4.1" max_tokens: 2000 suppress_comments: ${{ env.SUPPRESS_COMMENTS }} suppress_labels: ${{ env.SUPPRESS_LABELS }} - name: Post-process triage results if: steps.ai-assessment.outputs.ai_assessments != '' uses: actions/github-script@v7 env: ASSESSMENT_OUTPUT: ${{ steps.ai-assessment.outputs.ai_assessments }} SUPPRESS_LABELS: ${{ env.SUPPRESS_LABELS }} ISSUE_NUMBER: ${{ steps.issue.outputs.number }} with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | const assessments = JSON.parse(process.env.ASSESSMENT_OUTPUT); const issueNumber = parseInt(process.env.ISSUE_NUMBER); const owner = context.repo.owner; const repo = context.repo.repo; const suppressLabels = process.env.SUPPRESS_LABELS === 'true'; let needsHumanReview = false; let addHelpWanted = false; let needsAuthorFeedback = false; let canAutoClose = false; for (const assessment of assessments) { const label = (assessment.assessmentLabel || '').toLowerCase(); // Check if the assessment requires human review if (label.includes('needs team review') || label.includes('needs maintainer input') || label.includes('potential bug') || label.includes('needs discussion')) { needsHumanReview = true; } // Check if feature should be tagged as help wanted if (label.includes('help wanted')) { addHelpWanted = true; } // Check if more information is needed from the issue author if (label.includes('needs author feedback') || label.includes('requires additional details')) { needsAuthorFeedback = true; } // Check if the AI fully resolved the issue (answered question, explained misconfiguration, redirected to docs) if (label.includes('answered') || label.includes('likely misconfiguration') || label.includes('redirect to docs')) { canAutoClose = true; } // Log for job summary core.info(`Prompt: ${assessment.prompt}, Label: ${assessment.assessmentLabel}`); } // Skip label changes in summary-only mode (Phase 3) if (suppressLabels) { core.info('Labels suppressed — logging decisions only.'); core.exportVariable('LABEL_DECISIONS', JSON.stringify({ needsHumanReview, addHelpWanted, needsAuthorFeedback, canAutoClose, assessmentLabels: assessments.map(a => a.assessmentLabel), })); return; } // Add 'help wanted' label if AI recommended community contribution if (addHelpWanted) { await github.rest.issues.addLabels({ owner, repo, issue_number: issueNumber, labels: ['help wanted'] }); core.info('Added "help wanted" label based on AI assessment.'); } // If AI fully handled the issue without needing team review if (!needsHumanReview) { // Auto-close if AI fully resolved (answered, misconfiguration, redirected) if (canAutoClose && !needsAuthorFeedback && !addHelpWanted) { await github.rest.issues.update({ owner, repo, issue_number: issueNumber, state: 'closed', state_reason: 'completed' }); core.info('Auto-closed issue — AI fully resolved it.'); } } else { // Add consolidated label for easy filtering of all issues needing team attention await github.rest.issues.addLabels({ owner, repo, issue_number: issueNumber, labels: ['ai:needs team attention'] }); // Notify team via comment on escalated issues await github.rest.issues.createComment({ owner, repo, issue_number: issueNumber, body: '🔔 @microsoft/fabric-cicd — This issue has been flagged by AI triage as requiring team attention. Please review the assessment above.' }); core.info('Escalated to team.'); } // Always remove 'needs triage' — triage is complete regardless of outcome try { await github.rest.issues.removeLabel({ owner, repo, issue_number: issueNumber, name: 'needs triage' }); core.info('Removed "needs triage" — triage complete.'); } catch (e) { core.info(`Could not remove "needs triage" label: ${e.message}`); } - name: Generate triage summary if: always() && steps.ai-assessment.outputs.ai_assessments != '' uses: actions/github-script@v7 env: ASSESSMENT_OUTPUT: ${{ steps.ai-assessment.outputs.ai_assessments }} LABEL_DECISIONS: ${{ env.LABEL_DECISIONS }} ISSUE_NUMBER: ${{ steps.issue.outputs.number }} ISSUE_TITLE: ${{ steps.issue.outputs.title }} ISSUE_URL: ${{ steps.issue.outputs.html_url }} with: script: | const assessments = JSON.parse(process.env.ASSESSMENT_OUTPUT); const issueNumber = process.env.ISSUE_NUMBER; const issueTitle = process.env.ISSUE_TITLE; const issueUrl = process.env.ISSUE_URL; let summary = `## 🤖 AI Triage Report\n\n`; summary += `**Issue:** [#${issueNumber} — ${issueTitle}](${issueUrl})\n\n`; for (const assessment of assessments) { summary += `### Prompt: \`${assessment.prompt}\`\n`; summary += `**Assessment:** \`${assessment.assessmentLabel}\`\n\n`; summary += `
Full AI Response\n\n`; summary += `${assessment.response}\n\n`; summary += `
\n\n`; } // Show label decisions const ld = process.env.LABEL_DECISIONS ? JSON.parse(process.env.LABEL_DECISIONS) : null; if (ld) { summary += `### 🏷️ Label Decisions (not applied — testing mode)\n\n`; summary += `| Decision | Value |\n|----------|-------|\n`; summary += `| AI assessment labels | ${(ld.assessmentLabels || []).map(l => '`' + l + '`').join(', ')} |\n`; summary += `| Would add \`help wanted\` | ${ld.addHelpWanted ? '✅ Yes' : '❌ No'} |\n`; summary += `| Would add \`ai:needs team attention\` | ${ld.needsHumanReview ? '✅ Yes' : '❌ No'} |\n`; summary += `| Would request author feedback | ${ld.needsAuthorFeedback ? '✅ Yes' : '❌ No'} |\n`; summary += `| Would remove \`needs triage\` | ✅ Yes (always) |\n`; summary += `| Would auto-close | ${ld.canAutoClose && !ld.needsAuthorFeedback && !ld.addHelpWanted ? '✅ Yes' : '❌ No'} |\n`; summary += `| Would notify team | ${ld.needsHumanReview ? '✅ Yes' : '❌ No'} |\n\n`; } summary += `---\n\n`; core.summary.addRaw(summary); await core.summary.write(); const fs = require('fs'); fs.mkdirSync('triage-reports', { recursive: true }); fs.writeFileSync( `triage-reports/issue-${issueNumber}-triage.md`, summary ); - name: Upload triage report if: always() && steps.ai-assessment.outputs.ai_assessments != '' uses: actions/upload-artifact@v4 with: name: triage-report-issue-${{ steps.issue.outputs.number }} path: triage-reports/ retention-days: 30 ================================================ FILE: .github/workflows/bump.yml ================================================ name: Bump Version on: pull_request: types: [closed] branches: - main workflow_dispatch: inputs: pr_number: description: "Pull Request number to create release from" required: true type: string permissions: contents: write pull-requests: read jobs: get-pr-details: name: Get PR Details runs-on: ubuntu-latest if: github.event_name == 'workflow_dispatch' outputs: pr_title: ${{ steps.pr-info.outputs.pr_title }} pr_head_sha: ${{ steps.pr-info.outputs.pr_head_sha }} steps: - name: Get PR Information id: pr-info uses: actions/github-script@v7 with: script: | const prNumber = parseInt('${{ github.event.inputs.pr_number }}'); const { data: pr } = await github.rest.pulls.get({ owner: context.repo.owner, repo: context.repo.repo, pull_number: prNumber }); console.log(`PR #${prNumber}: ${pr.title}`); console.log(`Merge commit SHA: ${pr.merge_commit_sha}`); core.setOutput('pr_title', pr.title); core.setOutput('pr_head_sha', pr.merge_commit_sha); validate-version-bump: name: Validate Version Bump runs-on: ubuntu-latest needs: [get-pr-details] if: always() && !cancelled() outputs: is_version_bump: ${{ steps.version_bump_check.outputs.is_version_bump }} steps: - name: Checkout code uses: actions/checkout@v4 with: fetch-depth: 0 - name: Validate Version Bump id: version_bump_check run: | set -e PR_TITLE="${{ github.event.pull_request.title || needs.get-pr-details.outputs.pr_title }}" VERSION_REGEX='^v([0-9]+\.[0-9]+\.[0-9]+)$' if [[ "$PR_TITLE" =~ $VERSION_REGEX ]]; then echo "✅ PR title matches version format: $PR_TITLE" echo "is_version_bump=true" >> $GITHUB_OUTPUT else echo "ℹ️ PR title does not match version format: $PR_TITLE" echo "is_version_bump=false" >> $GITHUB_OUTPUT fi bump-version: name: Bump Version needs: [validate-version-bump] if: | (github.event_name == 'workflow_dispatch') || (github.event_name == 'pull_request' && github.event.pull_request.merged == true && needs.validate-version-bump.outputs.is_version_bump == 'true') runs-on: ubuntu-latest steps: - name: Generate GitHub App Token id: app-token uses: actions/create-github-app-token@v1 with: app-id: ${{ secrets.APP_ID }} private-key: ${{ secrets.APP_PRIVATE_KEY }} - name: Checkout repository uses: actions/checkout@v4 with: fetch-depth: 0 token: ${{ steps.app-token.outputs.token }} - name: Extract version id: version-check run: | # Determine which commit to get version from if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then COMMIT_SHA="${{ needs.get-pr-details.outputs.pr_head_sha }}" git fetch origin "$COMMIT_SHA" CONSTANTS_VERSION=$(git show ${COMMIT_SHA}:src/fabric_cicd/constants.py | grep -oP '(?<=^VERSION = ").*(?=")') else # For pull_request events, use current working directory (HEAD) CONSTANTS_VERSION=$(grep -oP '(?<=^VERSION = ").*(?=")' src/fabric_cicd/constants.py) fi echo "Version: $CONSTANTS_VERSION" if [ -z "$CONSTANTS_VERSION" ]; then echo "ERROR: Could not extract version from constants.py" exit 1 fi echo "version_number=$CONSTANTS_VERSION" >> $GITHUB_OUTPUT echo "version_tag=v$CONSTANTS_VERSION" >> $GITHUB_OUTPUT # Log trigger type if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then echo "🚀 Manual trigger for PR #${{ github.event.inputs.pr_number }}" elif [ "${{ github.event_name }}" = "pull_request" ]; then echo "🚀 PR merge trigger - validated version bump" fi - name: Create tag if needed if: needs.validate-version-bump.outputs.is_version_bump == 'true' id: tag-creation run: | TAG_NAME="${{ steps.version-check.outputs.version_tag }}" echo "Checking tag: $TAG_NAME" # Check if tag exists locally or remotely if git tag -l | grep -q "^$TAG_NAME$" || git ls-remote --tags origin | grep -q "refs/tags/$TAG_NAME$"; then echo "ℹ️ Tag $TAG_NAME already exists, will proceed with existing tag" echo "tag_created=false" >> $GITHUB_OUTPUT else echo "✅ Tag $TAG_NAME does not exist, creating new tag" # Configure git git config user.name "fabric-cicd-release[bot]" git config user.email "fabric-cicd-release[bot]@users.noreply.github.com" # Create and push the tag git tag "$TAG_NAME" git push origin "$TAG_NAME" echo "✅ Created and pushed tag: $TAG_NAME" echo "tag_created=true" >> $GITHUB_OUTPUT fi - name: Create GitHub Release if: needs.validate-version-bump.outputs.is_version_bump == 'true' uses: actions/github-script@v7 with: github-token: ${{ steps.app-token.outputs.token }} script: | const tagName = '${{ steps.version-check.outputs.version_tag }}'; try { // Check if release already exists let existingRelease; try { existingRelease = await github.rest.repos.getReleaseByTag({ owner: context.repo.owner, repo: context.repo.repo, tag: tagName }); console.log(`ℹ️ Release ${tagName} already exists: ${existingRelease.data.html_url}`); console.log('Skipping release creation'); return; } catch (error) { if (error.status !== 404) { throw error; } // Release doesn't exist, proceed to create it } // Extract changelog content for new release const versionNumber = '${{ steps.version-check.outputs.version_number }}'; const fs = require('fs'); const path = 'docs/changelog.md'; let changelogContent = `Release ${tagName}`; if (fs.existsSync(path)) { try { const changelogText = fs.readFileSync(path, 'utf8'); const lines = changelogText.split('\n'); let found = false; let content = []; for (const line of lines) { if (line.startsWith('## [v')) { if (found) break; // Hit next version, stop if (line.includes(`[v${versionNumber}]`)) { found = true; continue; // Skip the version header line } } else if (found) { // Skip span tags and empty lines if (line.trim() === '' || line.startsWith(' 0) { changelogContent = content.join('\n').trim(); } else { console.log(`Changelog content not found for version ${versionNumber}`); } } catch (error) { console.log('Error reading changelog:', error.message); } } else { console.log('Changelog file not found'); } // Create new release const release = await github.rest.repos.createRelease({ owner: context.repo.owner, repo: context.repo.repo, tag_name: tagName, name: tagName, body: changelogContent, draft: false, prerelease: false }); console.log(`✅ Created GitHub release: ${tagName}`); console.log(`Release URL: ${release.data.html_url}`); } catch (error) { console.error('Failed to create release:', error); throw error; } ================================================ FILE: .github/workflows/changelog.yml ================================================ # yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json --- name: 🔄 Changelog on: pull_request: branches: - main types: - opened - reopened - labeled - unlabeled - synchronize workflow_dispatch: concurrency: group: ${{ format('{0}-{1}-{2}-{3}-{4}', github.workflow, github.event_name, github.ref, github.base_ref || null, github.head_ref || null) }} cancel-in-progress: true permissions: contents: read jobs: changelog-existence: name: 🔄 Check Changelog if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip changelog') && github.actor != 'dependabot[bot]' }} runs-on: ubuntu-latest env: BASE_SHA: ${{ github.event.pull_request.base.sha }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} steps: - name: ⤵️ Checkout uses: actions/checkout@v4 with: fetch-depth: 0 # Required to access full commit history - name: ✔️ Check for changelog changes id: changelog_check uses: actions/github-script@v6 with: script: | const { execSync } = require('child_process'); const base = process.env.BASE_SHA; const head = process.env.HEAD_SHA; console.log(`Comparing changes from ${base} to ${head}`) const output = execSync(`git diff --name-only --no-renames --diff-filter=AM ${base} ${head}`).toString(); const files = output.split('\n').filter(Boolean); const changelogExists = files.some(file => file.startsWith('.changes/unreleased/') && file.endsWith('.yaml')); core.setOutput('exists', changelogExists); - name: 🚧 Setup Node if: steps.changelog_check.outputs.exists == 'true' uses: actions/setup-node@v6 with: node-version: "20" - name: 🚧 Install Changie if: steps.changelog_check.outputs.exists == 'true' run: npm i -g changie - name: 🔄 Prepare comment (changelog) if: steps.changelog_check.outputs.exists == 'true' run: | echo -e "# Changelog Preview\n" > changie.md changie batch patch --dry-run --prerelease 'dev' >> changie.md cat changie.md >> $GITHUB_STEP_SUMMARY - name: 🔄 Prepare comment (missing) if: steps.changelog_check.outputs.exists == 'false' run: | echo -e "# 🛑 Changelog entry required to merge\n" > changie.md echo "Run \`changie new\` to add a new changelog entry" >> changie.md cat changie.md >> $GITHUB_STEP_SUMMARY - name: ✅ Pass if changelog entry exists if: steps.changelog_check.outputs.exists == 'true' run: | echo "✅ Changelog entry exists." exit 0 - name: 🛑 Fail if changelog entry is missing and required if: steps.changelog_check.outputs.exists == 'false' run: | echo "🛑 Changelog entry required to merge." echo "🛑 Please run 'changie new' to add a new changelog entry." exit 1 changelog-skip: name: ⏭️ Skip Changelog if: ${{ contains(github.event.pull_request.labels.*.name, 'skip changelog') || github.actor == 'dependabot[bot]' }} runs-on: ubuntu-latest steps: - name: ✅ Pass (skip) run: | echo "⏭️ Changelog check skipped." >> $GITHUB_STEP_SUMMARY exit 0 ================================================ FILE: .github/workflows/publish_docs.yml ================================================ name: Publish Docs on: workflow_run: workflows: ["Bump Version"] types: [completed] workflow_dispatch: permissions: contents: write jobs: publish-docs: name: Publish Docs runs-on: ubuntu-latest if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' steps: - name: Generate GitHub App Token id: app-token uses: actions/create-github-app-token@v1 with: app-id: ${{ secrets.APP_ID }} private-key: ${{ secrets.APP_PRIVATE_KEY }} - name: Checkout repository uses: actions/checkout@v4 with: fetch-depth: 0 token: ${{ steps.app-token.outputs.token }} - name: Extract version id: version-check run: | CONSTANTS_VERSION=$(grep -oP '(?<=^VERSION = ").*(?=")' src/fabric_cicd/constants.py) echo "Version: $CONSTANTS_VERSION" if [ -z "$CONSTANTS_VERSION" ]; then echo "ERROR: Could not extract version from constants.py" exit 1 fi echo "version_number=$CONSTANTS_VERSION" >> $GITHUB_OUTPUT - name: Install Python uses: actions/setup-python@v5 with: python-version: "3.9" - name: Install Requirements run: | python -m pip install --upgrade pip python -m pip install uv - name: Deploy GitHub Pages env: GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} run: | git config user.name "fabric-cicd-release[bot]" git config user.email "fabric-cicd-release[bot]@users.noreply.github.com" git fetch --no-tags --prune --depth=1 origin +refs/heads/gh-pages:refs/remotes/origin/gh-pages uv sync VERSION="${{ steps.version-check.outputs.version_number }}" echo "Deploying docs for version: $VERSION" uv run mike deploy \ --update-aliases \ --branch gh-pages \ --push \ $VERSION \ latest uv run mike set-default --push latest ================================================ FILE: .github/workflows/test.yml ================================================ name: Test description: "Run unit tests for the Fabric CICD project" on: pull_request: branches: ["main"] types: [opened, edited, synchronize, ready_for_review] push: branches: ["main"] jobs: unit_test: name: Unit Test (Python ${{ matrix.python-version }}) runs-on: ubuntu-latest strategy: fail-fast: false matrix: python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] steps: - name: Checkout code uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Run unit tests run: | python -m pip install --upgrade pip pip install uv uv sync --dev uv run pytest -v || exit 1 # Fail the job if any tests fail ================================================ FILE: .github/workflows/validate.yml ================================================ name: Validate PR description: "Validate pull requests for code conventions, naming conventions, linked issues, and version bumps" on: pull_request: branches: ["main"] types: [opened, edited, synchronize, ready_for_review, labeled, unlabeled] permissions: contents: read pull-requests: write issues: write statuses: write jobs: format_ruff: if: ${{ github.event.action != 'labeled' && github.event.action != 'unlabeled' }} name: Code Formatted runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v2 with: python-version: "3.x" - name: Install dependencies run: | python -m pip install --upgrade pip pip install ruff - name: Run ruff format run: | if ruff format; then echo "✅ ruff format passed." else echo "❌ ruff format failed." exit 1 fi lint_ruff: if: ${{ github.event.action != 'labeled' && github.event.action != 'unlabeled' }} name: Code Linted runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v2 with: python-version: "3.x" - name: Install dependencies run: | python -m pip install --upgrade pip pip install ruff - name: Run ruff lint run: | if ruff check; then echo "✅ ruff lint passed." else echo "❌ ruff lint failed." exit 1 fi validate-version-bump: name: Proper Version Bump runs-on: ubuntu-latest outputs: is_version_bump: ${{ steps.version_bump_check.outputs.is_version_bump }} steps: - name: Checkout code uses: actions/checkout@v4 with: fetch-depth: 0 - name: Validate Version Bump id: version_bump_check env: PR_TITLE: ${{ github.event.pull_request.title }} PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | set -e VERSION_REGEX='^v([0-9]+\.[0-9]+\.[0-9]+)$' BASE_REF="origin/main" CHANGED_FILES=$(git diff --name-only "$BASE_REF...$PR_HEAD_SHA") VERSION_CHANGED=false TITLE_IS_VERSION=false OLD_VERSION="" NEW_VERSION="" # Get VERSION from main branch OLD_VERSION=$(git show origin/main:src/fabric_cicd/constants.py | grep '^VERSION = ' | sed -E 's/VERSION = "([^"]+)"/\1/') # Get VERSION from current branch NEW_VERSION=$(grep '^VERSION = ' src/fabric_cicd/constants.py | sed -E 's/VERSION = "([^"]+)"/\1/') if [ "$OLD_VERSION" != "$NEW_VERSION" ]; then VERSION_CHANGED=true fi if [[ "$PR_TITLE" =~ $VERSION_REGEX ]]; then TITLE_IS_VERSION=true fi if [ "$TITLE_IS_VERSION" = true ] || [ "$VERSION_CHANGED" = true ]; then echo "is_version_bump=true" >> $GITHUB_OUTPUT else echo "is_version_bump=false" >> $GITHUB_OUTPUT echo "✅ Version bump validation passed." exit 0 fi # 1. If version is updated, title must match if [ "$VERSION_CHANGED" = true ]; then EXPECTED_TITLE="v$NEW_VERSION" if [ "$PR_TITLE" != "$EXPECTED_TITLE" ]; then echo "❌ PR title must be vX.X.X when VERSION is changed. Expected title: $EXPECTED_TITLE" exit 1 fi fi # 2. If title is version format, constants.py, changelog.md, and .changes/v.md must be included in changed files # Note: Additional files (e.g., removed unreleased change files) are allowed alongside the required files if [[ "$PR_TITLE" =~ $VERSION_REGEX ]]; then VERSION_NUMBER="${BASH_REMATCH[1]}" REQUIRED_FILES=("src/fabric_cicd/constants.py" "docs/changelog.md" ".changes/v${VERSION_NUMBER}.md") MISSING_FILES=() for file in "${REQUIRED_FILES[@]}"; do if ! echo "$CHANGED_FILES" | grep -Fqx "$file"; then MISSING_FILES+=("$file") fi done if [ ${#MISSING_FILES[@]} -ne 0 ]; then echo "❌ The following required files must be included in a PR titled vX.X.X: ${MISSING_FILES[*]}" exit 1 fi fi echo "✅ Version bump validation passed." check-author-permissions: name: Check Author Permissions runs-on: ubuntu-latest outputs: skip_validation: ${{ steps.check_permissions.outputs.skip_validation }} permissions: pull-requests: read steps: - name: Check PR Author Collaborator Role id: check_permissions uses: actions/github-script@v7 with: script: | const prAuthor = context.payload.pull_request.user.login; const repo = context.repo; // Skip validation for dependabot if (prAuthor === 'dependabot[bot]') { console.log('✅ Dependabot PR detected. Skipping linked issue validation.'); core.setOutput('skip_validation', 'true'); return; } console.log(`Checking collaborator role for PR author: ${prAuthor}`); try { const { data: collaborator } = await github.rest.repos.getCollaboratorPermissionLevel({ owner: repo.owner, repo: repo.repo, username: prAuthor }); const roleName = collaborator.role_name; console.log(`Collaborator role for ${prAuthor}: ${roleName}`); // Skip validation for users with admin, maintain, or write role const skipValidation = ['admin', 'maintain', 'write'].includes(roleName); core.setOutput('skip_validation', skipValidation.toString()); if (skipValidation) { console.log(`✅ User ${prAuthor} has ${roleName} role. Validation job for linked issue will be skipped.`); } else { console.log(`ℹ️ User ${prAuthor} has ${roleName} role. Validation job for linked issue will run.`); } } catch (error) { console.log(`⚠️ Could not determine collaborator role for ${prAuthor}: ${error.message}`); console.log('Defaulting to running validation job for linked issue.'); core.setOutput('skip_validation', 'false'); } validate-linked-issue: name: Issue Linked runs-on: ubuntu-latest needs: [check-author-permissions, validate-version-bump] if: needs.check-author-permissions.outputs.skip_validation == 'false' && needs.validate-version-bump.outputs.is_version_bump == 'false' && !contains(github.event.pull_request.labels.*.name, 'skip changelog') permissions: pull-requests: read issues: read steps: - name: Validate Linked Issue uses: actions/github-script@v7 with: script: | const prNumber = context.issue.number; const repo = context.repo; // Get PR details const pr = await github.rest.pulls.get({ owner: repo.owner, repo: repo.repo, pull_number: prNumber }); const prTitle = pr.data.title || ''; // First, check for issues linked to the PR via GitHub's native linking let linkedIssues = []; try { // Use GraphQL to get linked issues const query = ` query($owner: String!, $repo: String!, $number: Int!) { repository(owner: $owner, name: $repo) { pullRequest(number: $number) { closingIssuesReferences(first: 10) { nodes { number } } } } } `; const result = await github.graphql(query, { owner: repo.owner, repo: repo.repo, number: prNumber }); linkedIssues = result.repository.pullRequest.closingIssuesReferences.nodes; if (linkedIssues.length > 0) { console.log(`✅ Found ${linkedIssues.length} linked issue(s): ${linkedIssues.map(issue => `#${issue.number}`).join(', ')}`); console.log('✅ Pull request is properly linked to an issue via GitHub linking.'); return; } } catch (error) { console.log('⚠️ Could not check for linked issues via GraphQL, falling back to text analysis:', error.message); } // Issue reference patterns - more specific to avoid false positives const keywordPatterns = [ /(?:close[sd]?|fix(?:e[sd])?|resolve[sd]?)\s+#(\d+)/gi, /(?:close[sd]?|fix(?:e[sd])?|resolve[sd]?)\s+https:\/\/github\.com\/[^\/]+\/[^\/]+\/issues\/(\d+)/gi ]; // Generic hash pattern (less specific, used as fallback) const hashPattern = /#(\d+)(?!\w)/g; let foundIssueNumbers = new Set(); // Check PR title for issue references const textToCheck = prTitle; // First, look for keyword-based references (more reliable) for (const pattern of keywordPatterns) { const matches = textToCheck.matchAll(pattern); for (const match of matches) { const issueNumber = match[1]; if (issueNumber && !isNaN(issueNumber)) { foundIssueNumbers.add(parseInt(issueNumber)); } } } // If no keyword-based references found, look for simple hash references if (foundIssueNumbers.size === 0) { const matches = textToCheck.matchAll(hashPattern); for (const match of matches) { const issueNumber = match[1]; if (issueNumber && !isNaN(issueNumber)) { foundIssueNumbers.add(parseInt(issueNumber)); } } } if (foundIssueNumbers.size === 0) { core.setFailed( '❌ This pull request must be linked to an issue. Please:\n' + '1. Reference an issue in the PR title using "Fixes #123", "Closes #456", or "Resolves #789"\n' + '2. Make sure the referenced issue exists in this repository\n\n' + 'See our contribution guidelines for more details.' ); return; } // Verify that the referenced issues actually exist let validIssueFound = false; const invalidIssues = []; for (const issueNumber of foundIssueNumbers) { try { await github.rest.issues.get({ owner: repo.owner, repo: repo.repo, issue_number: issueNumber }); validIssueFound = true; console.log(`✅ Found valid issue reference: #${issueNumber}`); } catch (error) { if (error.status === 404) { invalidIssues.push(issueNumber); console.log(`❌ Issue #${issueNumber} does not exist`); } } } if (!validIssueFound) { const invalidList = invalidIssues.length > 0 ? `\n\nInvalid issue references found: ${invalidIssues.map(n => `#${n}`).join(', ')}` : ''; core.setFailed( '❌ This pull request must be linked to a valid issue in this repository.' + invalidList + '\n\nPlease:\n' + '1. Create an issue first if one doesn\'t exist\n' + '2. Reference the issue in the PR title using "Fixes #123", "Closes #456", or "Resolves #789"\n' + '3. Make sure the issue number is correct\n\n' + 'See our contribution guidelines for more details.' ); return; } console.log('✅ Pull request is properly linked to an issue.'); ================================================ FILE: .gitignore ================================================ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ cover/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder .pybuilder/ target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv # For a library or package, you might want to ignore these files since the code is # intended to run in multiple environments; otherwise, check them in: # .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # poetry # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. # This is especially recommended for binary packages to ensure reproducibility, and is more # commonly ignored for libraries. # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control #poetry.lock # pdm # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. #pdm.lock # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it # in version control. # https://pdm.fming.dev/#use-with-ide .pdm.toml # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # ruff .ruff_cache/ # Pyre type checker .pyre/ # pytype static type analyzer .pytype/ # Cython debug symbols cython_debug/ # PyCharm # JetBrains specific template is maintained in a separate JetBrains.gitignore that can # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ # http traces should only be committed at the fixture root /http_trace.json /http_trace.json.lock /http_trace.json.gz ================================================ FILE: .prettierignore ================================================ sample/workspace/ docs/how_to/parameterization.md ================================================ FILE: .prettierrc ================================================ { "printWidth": 100, "tabWidth": 4, "useTabs": false, "semi": true, "trailingComma": "all" } ================================================ FILE: .python-version ================================================ 3.11 ================================================ FILE: .vscode/extensions.json ================================================ { "recommendations": [ "ms-python.python", "esbenp.prettier-vscode", "ms-vscode.powershell", "charliermarsh.ruff", "tamasfe.even-better-toml" ] } ================================================ FILE: .vscode/launch.json ================================================ { "version": "0.2.0", "configurations": [ { "name": "Debug: Trace Publish All Items", "type": "debugpy", "request": "launch", "program": "${workspaceFolder}/devtools/debug_trace_deployment.py", "console": "integratedTerminal", "justMyCode": false, "env": { "PYTHONPATH": "${workspaceFolder}/src", "FABRIC_WORKSPACE_ID": "your-fabric-workspace-guid" }, "cwd": "${workspaceFolder}" } ] } ================================================ FILE: .vscode/settings.json ================================================ { "editor": { "trimAutoWhitespace": false, "defaultFormatter": "esbenp.prettier-vscode", "formatOnSave": true }, "diffEditor": { "ignoreTrimWhitespace": false }, "files": { "trimTrailingWhitespace": false, "trimTrailingWhitespaceInRegexAndStrings": false, "insertFinalNewline": true, "autoSave": "off" }, "[python]": { "editor.defaultFormatter": "charliermarsh.ruff", "editor.codeActionsOnSave": { "source.fixAll": "explicit" }, "editor.formatOnSave": true }, "[powershell]": { "editor.defaultFormatter": "ms-vscode.powershell", "editor.formatOnSave": true }, "[toml]": { "editor.defaultFormatter": "tamasfe.even-better-toml", "editor.formatOnSave": true }, "git": { "branchPrefix": "users/", "enableSmartCommit": true, "confirmSync": false, "autofetch": true }, "ruff": { "organizeImports": true, "fixAll": true }, "python.terminal.activateEnvironment": false, "python.testing.pytestArgs": ["tests"], "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true } ================================================ FILE: CODE_OF_CONDUCT.md ================================================ # Microsoft Open Source Code of Conduct This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). Resources: - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns ================================================ FILE: CONTRIBUTING.md ================================================ # Contributing This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit https://cla.microsoft.com. When you submit a pull request, a CLA-bot will automatically determine whether you need to provide a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions provided by the bot. You will only need to do this once across all repositories using our CLA. This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. ## Ways to Contribute We welcome several types of contributions: - 🔧 **Bug fixes** - Fix issues and improve reliability - ✨ **New features** - Add new commands or functionality - 🆕 **New Items Support** - Onboard new Fabric item types - 📝 **Documentation** - Improve guides, examples, and API docs - 🧪 **Tests** - Add or improve test coverage - 💬 **Help others** - Answer questions and provide support - 💡 **Feature suggestions** - Propose new capabilities ## Prerequisites Before you begin, ensure you have the following installed: - [Python](https://www.python.org/downloads/) (see [Installation](https://microsoft.github.io/fabric-cicd/#installation) for version requirements) - [Node.js and npm](https://nodejs.org/en/download/) - [PowerShell](https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell) - [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli-windows) or [Az.Accounts PowerShell module](https://www.powershellgallery.com/packages/Az.Accounts/2.2.3) - [Visual Studio Code (VS Code)](https://code.visualstudio.com/) ## Initial Configuration 1. **Fork the Repository on GitHub**: - Go to the repository [fabric-cicd](https://github.com/microsoft/fabric-cicd) on GitHub - In the top right corner, click on the **Fork** button - This will create a copy of the repository in your own GitHub account 1. **Clone Your Forked Repository**: - Once the fork is complete, go to your GitHub account and open the forked repository - Click on the **Code** button, and clone to VS Code 1. **Run activate.ps1**: - Open the Project in VS Code - Open PowerShell terminal - Run `activate.ps1` which will install `uv`, and `ruff` if not already found. And set up the default environment leveraging `uv sync` ```powershell .\activate.ps1 ``` _Note that this is technically optional and is designed to work with PowerShell. You can execute these steps manually as well, this is merely a helper_ _For Linux, run `activate.sh` instead_ 1. **Select Python Interpreter**: - Open the Command Palette (`Ctrl+Shift+P`) and select `Python: Select Interpreter` - Choose the interpreter from the `.venv` directory 1. **Ensure All VS Code Extensions Are Installed**: - Open the Command Palette (`Ctrl+Shift+P`) and select `Extensions: Show Recommended Extensions` - Install all extensions recommended for the workspace ## Development ### Managing Dependencies - All dependencies in this project are managed by `uv` which will resolve all dependencies and lock the versions to speed up virtual environment creation - For additions, run: ```sh uv add ``` - For removals, run: ```sh uv remove ``` ### Code Formatting & Linting - The python code within this project is maintained by `ruff` - If you install the recommended extensions, `ruff` will auto format on save of any file - Before being able to merge a PR, `ruff` is ran in a GitHub Action to ensure the files are properly formatted and maintained - To force linting, run the following ```sh uv run ruff format uv run ruff check ``` ## Contribution process To avoid cases where submitted PRs are rejected, please follow the following steps: - To report a new issue, follow [Create an issue](#creating-an-issue) - To work on existing issue, follow [Find an issue to work on](#finding-an-issue-to-work-on) - To contribute code, follow [Pull request process](#pull-request-process) ### Creating an issue Before reporting a new bug or suggesting a feature, please search the [GitHub Issues page](https://github.com/microsoft/fabric-cicd/issues) to check if one already exists. All reported bugs or feature suggestions must start with creating an issue in the GitHub Issues pane. Please add as much information as possible to help us with triage and understanding. Once the issue is triaged, labels will be added to indicate its status (e.g., "need more info", "help wanted"). When creating an issue please select the relevant template, e.g., bug, new feature, general question, etc. and provide all required input: - [Bug Report](https://github.com/microsoft/fabric-cicd/issues/new?template=1-bug.yml) - [Feature Request](https://github.com/microsoft/fabric-cicd/issues/new?template=2-feature.yml) - [Documentation](https://github.com/microsoft/fabric-cicd/issues/new?template=3-documentation.yml) - [Question](https://github.com/microsoft/fabric-cicd/issues/new?template=4-question.yml) We aim to respond to new issues promptly, but response times may vary depending on workload and priority. ### Finding an issue to work on #### For Beginners If you're new to contributing, look for issues with these labels: - **`good-first-issue`** - Beginner-friendly tasks that are well-scoped and documented - **`help wanted`** - Issues where community contributions are especially welcome - **`documentation`** - Improve docs, examples, or help text (great for first contributions) #### Getting Started Tips 1. **Start small** - Look for typo fixes, documentation improvements, or simple bug fixes 2. **Read existing code** - Familiarize yourself with the codebase by exploring similar commands 3. **Ask questions** - Comment on issues to clarify requirements or get guidance 4. **Test locally** - Always test your changes thoroughly before submitting #### Before You Code All PRs must be linked with a "help wanted" issue. To avoid rework after investing effort: 1. **Comment on the issue** - Express interest and describe your planned approach 2. **Wait for acknowledgment** - Get team confirmation before starting significant work 3. **Ask for clarification** - Don't hesitate to ask questions about requirements Please review [engineering guidelines](https://github.com/microsoft/fabric-cicd/wiki) for coding guidelines and common flows to help you with your task. ### Pull request process **All pull requests must be linked to an approved issue,** see [PR Title Format](#pr-title-format). This ensures proper tracking and context for changes. Before creating a pull request: 1. **Create or identify an existing issue** that describes the problem, feature request, or change you're addressing 2. **Comment on the issue** to express interest and get team acknowledgment before starting work #### PR Title Format Your PR title MUST follow this exact format: `"Fixes #123 - Short Description"` where #123 is the issue number. - Use "Fixes" for bug fixes, "Closes" for features, "Resolves" for other changes - Example: "Fixes #520 - Add Python version requirements to documentation" - Version bump PRs are an exception: title must be "vX.X.X" format only - GitHub Actions will automatically check that your PR is linked to a valid issue and will fail if no valid reference is found #### Before Submitting PR Verify that: - The PR is focused on the related task - Tests coverage is kept and all tests pass - Your code is aligned with the code conventions of this project #### Review Process - Use a descriptive title and provide a clear summary of your changes - Address and resolve all review comments before merge - PRs will be labeled as "need author feedback" when there are comments to resolve - Approved PRs will be merged by the fabric-cicd team ### Documenting Changes with Changie All pull requests must include proper change documentation using [changie](https://changie.dev), which is pre-installed in the development container. This ensures that release notes are automatically generated and changes are properly tracked. #### Requirements **Every PR must include at least one change entry** created using `changie new`. You may add multiple entries if your PR introduces multiple distinct changes. #### How to Add Change Entries 1. **From the Terminal, run `changie new` command**: ```bash changie new ``` 2. **Select the appropriate change type** from the available options: - **⚠️ Breaking Change** - For changes that break backward compatibility - **🆕 New Items Support** - For adding support for new Fabric item types - **✨ New Functionality** - For new features, commands, or capabilities - **🔧 Bug Fix** - For fixing existing issues or incorrect behavior - **⚡ Additional Optimizations** - For performance improvements or optimizations - **📝 Documentation Update** - For documentation improvements or updates 3. **Provide a clear description** of your change: - Write in present tense (e.g., "Add support for..." not "Added support for...") - Be specific and user-focused - Include the affected command or feature if applicable - Keep it concise but informative #### Examples of Good Change Descriptions - `Fix timeout issue in LRO polling` - `Update workspace examples with new folder hierarchy patterns` - `Optimize API response caching to reduce network calls` #### Guidelines - **One logical change per entry**: If your PR fixes a bug and adds a feature, create two separate entries - **User-facing perspective**: Describe what users will experience, not internal implementation details - **Clear and actionable**: Users should understand what changed and how it affects them - **Consistent formatting**: Follow the examples and existing patterns in the changelog The change entries will be automatically included in the release notes when a new version is published. This process ensures that all improvements, fixes, and new features are properly communicated to users. ## Resources to help you get started Here are some resources to help you get started: - A good place to start learning about fabric-cicd is the [fabric-cicd documentation](https://microsoft.github.io/fabric-cicd/) - If you want to contribute code, please check more details about coding guidelines, major code flows and code building block in [Engineering guidelines](https://github.com/microsoft/fabric-cicd/wiki) ## Engineering guidelines For detailed engineering guidelines please refer to our [Wiki pages](https://github.com/microsoft/fabric-cicd/wiki). The Wiki contains essential information and requirements for contributors, including: Code Style and Standards, Architecture Overview, Testing and more. Before contributing code, please review these guidelines to ensure your contributions align with the project's standards and practices. ## Areas with Restricted Contributions Some areas require special consideration: - **Core infrastructure** - Major architectural changes require team discussion, including within `FabricEndpoint` and `FabricWorkspace` classes - **Parameterization framework** - Changes require team discussion due to complex validation and parameter replacement logic ## Need Help? ### Getting Support - **[GitHub Issues](https://github.com/microsoft/fabric-cicd/issues)** - Report specific problems - **[Documentation](https://microsoft.github.io/fabric-cicd/)** - Check comprehensive guides ### Communication Guidelines - **Be patient** - Maintainers balance multiple responsibilities - **Be respectful** - Follow the code of conduct - **Be specific** - Provide clear, detailed information - **Be collaborative** - Work together to improve the project Thank you for contributing to Microsoft fabric-cicd! Your contributions help make this tool better for the entire Fabric community. ================================================ FILE: CodeQL.yml ================================================ path_classifiers: tests: - "devtools/*.py" ================================================ FILE: LICENSE ================================================ MIT License Copyright (c) Microsoft Corporation. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE ================================================ FILE: README.md ================================================ # Fabric CICD [![Language](https://img.shields.io/badge/language-Python-blue.svg)](https://www.python.org/) [![PyPi version](https://badgen.net/pypi/v/fabric-cicd/)](https://pypi.org/project/fabric-cicd) [![Python version](https://img.shields.io/pypi/pyversions/fabric-cicd)](https://pypi.org/project/fabric-cicd) [![Code style: ruff](https://img.shields.io/badge/code%20style-ruff-000000.svg)](https://github.com/charliermarsh/ruff) [![Tests](https://img.shields.io/github/actions/workflow/status/microsoft/fabric-cicd/test.yml?logo=github&label=tests&branch=main)](https://github.com/microsoft/fabric-cicd/actions/workflows/test.yml) --- ## Project Overview fabric-cicd is a Python library designed for use with [Microsoft Fabric](https://learn.microsoft.com/en-us/fabric/) workspaces. This library supports code-first Continuous Integration / Continuous Deployment (CI/CD) automations to seamlessly integrate Source Controlled workspaces into a deployment framework. The goal is to assist CI/CD developers who prefer not to interact directly with the Microsoft Fabric APIs. ## Documentation All documentation is hosted on our [fabric-cicd](https://microsoft.github.io/fabric-cicd/) GitHub Pages Section Overview: - [Home](https://microsoft.github.io/fabric-cicd/latest/) - [How To](https://microsoft.github.io/fabric-cicd/latest/how_to/) - [Examples](https://microsoft.github.io/fabric-cicd/latest/example/) - [Contribution](https://microsoft.github.io/fabric-cicd/latest/contribution/) - [Changelog](https://microsoft.github.io/fabric-cicd/latest/changelog/) - [About](https://microsoft.github.io/fabric-cicd/latest/help/) - Inclusive of Support & Security Policies ## Installation To install fabric-cicd, run: ```bash pip install fabric-cicd ``` ## Trademarks This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft trademarks or logos is subject to and must follow [Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general). Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. Any use of third-party trademarks or logos are subject to those third-party's policies. ================================================ FILE: SECURITY.md ================================================ ## Security Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet) and [Xamarin](https://github.com/xamarin). If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/security.md/definition), please report it to us as described below. ## Reporting Security Issues **Please do not report security vulnerabilities through public GitHub issues.** Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/security.md/msrc/create-report). If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/security.md/msrc/pgp). You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: - Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) - Full paths of source file(s) related to the manifestation of the issue - The location of the affected source code (tag/branch/commit or direct URL) - Any special configuration required to reproduce the issue - Step-by-step instructions to reproduce the issue - Proof-of-concept or exploit code (if possible) - Impact of the issue, including how an attacker might exploit the issue This information will help us triage your report more quickly. If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/security.md/msrc/bounty) page for more details about our active programs. ## Preferred Languages We prefer all communications to be in English. ## Policy Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/security.md/cvd). ================================================ FILE: activate.ps1 ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. <# .SYNOPSIS Script to check and install required Python packages, Node.js tools, add directories to PATH, and activate a virtual environment. .DESCRIPTION This script performs the following tasks: 1. Checks if Python is installed. 2. Checks if pip is installed. 3. Checks if Node.js and npm are installed. 4. Checks and installs specified Python packages if they are not already installed. 5. Installs changie globally via npm if not already installed. 6. Adds a specified directory to the system PATH if it is not already included. 7. Ensures the 'uv' command is available in the PATH. 8. Activates a virtual environment using 'uv'. .NOTES Make sure that Python, pip, Node.js, and npm are installed and available in the system PATH. #> # Function to check if a dependency is available function Test-Dependancy { param ( [string]$commandName ) if (-not (Get-Command $commandName -ErrorAction SilentlyContinue)) { Write-Host " $commandName is not installed or not in PATH. Please install $commandName and make sure it's available in the PATH." exit 1 } else { $commandPath = (Get-Command $commandName).Path $commandDirectory = [System.IO.Path]::GetDirectoryName($commandPath) Write-Host "$commandName is installed in $commandPath" Add-DirectoryToPath -directory $commandDirectory } } # Function to install required packages if not already installed function Test-And-Install-Python-Package { param ( [string]$packageName ) pip show $packageName -q if ($LASTEXITCODE -ne 0) { Write-Host "$packageName is not installed. Installing $packageName..." try { pip install $packageName Write-Host "$packageName installed successfully." } catch { Write-Host "Failed to install $packageName. Please check your pip installation." exit 1 } } else { Write-Host "$packageName is already installed." } } # Function to install changie globally via npm if not already installed function Test-And-Install-Changie { if (-not (Get-Command changie -ErrorAction SilentlyContinue)) { Write-Host "changie not found, installing globally via npm..." try { npm install -g changie --registry https://registry.npmjs.org/ # Add npm global bin to PATH if needed $npmGlobalPath = npm config get prefix if ($npmGlobalPath) { Add-DirectoryToPath -directory $npmGlobalPath } # Refresh PATH for the current session $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User") Test-Dependancy -commandName "changie" } catch { Write-Host "Failed to install changie via npm. Please check your npm installation and connection." exit 1 } } else { Write-Host "changie is already installed." } } # Function to add a directory to PATH function Add-DirectoryToPath { param ( [string]$directory ) if (-not ($env:Path -split ';' | ForEach-Object { $_.Trim() } | Where-Object { $_ -eq $directory })) { $env:Path += ";$directory" Write-Host "Added $directory to PATH." } } # Check if dependencies are installed and add directory to PATH Test-Dependancy -commandName "python" Test-Dependancy -commandName "pip" Test-Dependancy -commandName "node" Test-Dependancy -commandName "npm" # Check and install required packages Test-And-Install-Python-Package -packageName "uv" Test-And-Install-Python-Package -packageName "ruff" Test-And-Install-Changie # uv fallback to default path if unavailable in python directory if (-not (Get-Command uv -ErrorAction SilentlyContinue)) { Write-Host "uv is not recognized. Attempting to add uv to PATH..." $localBinPath = [System.IO.Path]::Combine($env:USERPROFILE, '.local', 'bin') Add-DirectoryToPath -directory $localBinPath Test-Dependancy -commandName "uv" } # Activate the environment uv sync --python 3.11 $venvPath = ".venv\Scripts\activate.ps1" if (Test-Path $venvPath) { & $venvPath Write-Host "venv activated" } else { Write-Host "venv not found" } Write-Host "" Write-Host "To deactivate the environment, run " -NoNewline Write-Host "deactivate" -ForegroundColor Green ================================================ FILE: activate.sh ================================================ #!/bin/bash # # # Script to check and install required Python packages, Node.js tools, # add directories to PATH, and activate a virtual environment. # # --------------------------------------------------------------------------------------- # set -e PACKAGES="" if ! command -v python3.11 &> /dev/null; then PACKAGES="python3.11"; fi if ! command -v pip &> /dev/null; then PACKAGES="${PACKAGES:+$PACKAGES }python3-pip"; fi if ! command -v node &> /dev/null; then PACKAGES="${PACKAGES:+$PACKAGES }nodejs"; fi if ! command -v npm &> /dev/null; then PACKAGES="${PACKAGES:+$PACKAGES }npm"; fi if [[ "$OSTYPE" == "linux-gnu"* ]]; then if [ -n "$PACKAGES" ]; then echo "Installing required packages for Linux: $PACKAGES" sudo apt-get update > /dev/null 2>&1 if sudo DEBIAN_FRONTEND=noninteractive apt-get install -y $PACKAGES > /dev/null 2>&1; then echo "Packages installed successfully." else echo "Failed to install packages." exit 1 fi fi elif [[ "$OSTYPE" == "darwin"* ]]; then if ! command -v brew &> /dev/null; then echo "Homebrew not found. Please install Homebrew first." exit 1 fi BREW_PACKAGES="" if ! command -v python3.11 &> /dev/null; then BREW_PACKAGES="python@3.11"; fi if ! command -v node &> /dev/null; then BREW_PACKAGES="${BREW_PACKAGES:+$BREW_PACKAGES }node"; fi if [ -n "$BREW_PACKAGES" ]; then echo "Installing required packages for macOS: $BREW_PACKAGES" if brew install $BREW_PACKAGES; then echo "Packages installed successfully." else echo "Failed to install packages." exit 1 fi fi fi # Install uv if not present if ! command -v uv &> /dev/null; then echo "Installing uv..." if curl -LsSf https://astral.sh/uv/install.sh | sh; then echo "uv installed successfully." else echo "Failed to install uv." exit 1 fi else echo "uv is already installed." fi # Install changie globally via npm if not present if ! command -v changie &> /dev/null; then echo "Installing changie globally via npm..." if npm install -g changie; then echo "changie installed successfully." else echo "Failed to install changie." exit 1 fi else echo "changie is already installed." fi # Install VS Code Python extension if VS Code is available if command -v code &> /dev/null; then echo "Installing VS Code Python extension..." if code --install-extension ms-python.python --force > /dev/null 2>&1; then echo "VS Code Python extension installed successfully." else echo "Failed to install VS Code Python extension." fi else echo "VS Code not found, skipping extension installation." fi # Add required directories to PATH if [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then export PATH="$PATH:$HOME/.local/bin" echo "Added $HOME/.local/bin to PATH." fi if [[ ":$PATH:" != *":$HOME/.cargo/bin:"* ]]; then export PATH="$PATH:$HOME/.cargo/bin" echo "Added $HOME/.cargo/bin to PATH." fi # Sync Python environment and activate echo "Syncing Python environment with uv..." if uv sync --python 3.11; then echo "Python environment synced successfully." else echo "Failed to sync Python environment." exit 1 fi if [ -f .venv/bin/activate ]; then source .venv/bin/activate echo "Virtual environment activated." else echo "Virtual environment not found." fi ================================================ FILE: devtools/debug_api.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # The following is intended for developers of fabric-cicd to debug and call Fabric REST APIs locally from the github repo from azure.identity import AzureCliCredential, AzurePowerShellCredential, ClientSecretCredential from fabric_cicd import change_log_level, constants from fabric_cicd._common._fabric_endpoint import FabricEndpoint from fabric_cicd._common._validate_input import validate_token_credential # Uncomment to enable debug # change_log_level() if __name__ == "__main__": # Azure CLI auth - comment out to use a different auth method token_credential = AzureCliCredential() # Uncomment to use PowerShell auth # token_credential = AzurePowerShellCredential() # Uncomment to use SPN auth # client_id = "your-client-id" # client_secret = "your-client-secret" # tenant_id = "your-tenant-id" # token_credential = ClientSecretCredential(client_id=client_id, client_secret=client_secret, tenant_id=tenant_id) # Create endpoint object fe = FabricEndpoint(token_credential=validate_token_credential(token_credential)) # Set workspace id variable if needed in API url workspace_id = "8f5c0cec-a8ea-48cd-9da4-871dc2642f4c" # API endpoint url (placeholder) api_url = f"{constants.DEFAULT_API_ROOT_URL}/v1/workspaces/{workspace_id}..." print("Making API call...") response = fe.invoke( method="POST", url=api_url, body={}, ) print("Call completed.") ================================================ FILE: devtools/debug_local config.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # The following is intended for developers of fabric-cicd to debug locally against the github repo import sys from pathlib import Path from azure.identity import AzureCliCredential, AzurePowerShellCredential, ClientSecretCredential root_directory = Path(__file__).resolve().parent.parent sys.path.insert(0, str(root_directory / "src")) from fabric_cicd import append_feature_flag, change_log_level, deploy_with_config # Uncomment to enable debug # change_log_level() # In this example, the config file sits within the root/sample/workspace directory config_file = str(root_directory / "sample" / "workspace" / "config.yml") # Azure CLI auth - comment out to use a different auth method token_credential = AzureCliCredential() # Uncomment to use PowerShell auth # token_credential = AzurePowerShellCredential() # Uncomment to use SPN auth # client_id = "your-client-id" # client_secret = "your-client-secret" # tenant_id = "your-tenant-id" # token_credential = ClientSecretCredential(client_id=client_id, client_secret=client_secret, tenant_id=tenant_id) # config_override_dict = {"core": {"item_types_in_scope": ["Notebook"]}, "publish": {"skip": {"dev": False}}} deploy_with_config( config_file_path=config_file, # Comment out if environment is not needed environment="dev", # Explicit token credential required for auth (choose one of the options above) token_credential=token_credential, # Uncomment to override specific config values (pass in a dictionary of override values) # config_override=config_override_dict ) ================================================ FILE: devtools/debug_local.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # The following is intended for developers of fabric-cicd to debug locally against the github repo import sys from pathlib import Path from azure.identity import AzureCliCredential, AzurePowerShellCredential, ClientSecretCredential root_directory = Path(__file__).resolve().parent.parent sys.path.insert(0, str(root_directory / "src")) from fabric_cicd import ( FabricWorkspace, append_feature_flag, change_log_level, constants, publish_all_items, unpublish_all_orphan_items, ) # Uncomment to enable debug # change_log_level() # Uncomment to add feature flag append_feature_flag("enable_shortcut_publish") # The defined environment values should match the names found in the parameter.yml file workspace_id = "8f5c0cec-a8ea-48cd-9da4-871dc2642f4c" environment = "PPE" # In this example, our workspace content sits within the root/sample/workspace directory repository_directory = str(root_directory / "sample" / "workspace") # Explicitly define which of the item types we want to deploy item_type_in_scope = [ "Lakehouse", "VariableLibrary", "DataBuildToolJob", "Dataflow", "DataPipeline", "Notebook", "Environment", "SemanticModel", "Report", "Eventhouse", "KQLDatabase", "KQLQueryset", "Reflex", "Eventstream", "SparkJobDefinition", "Ontology", ] # Azure CLI auth - comment out to use a different auth method token_credential = AzureCliCredential() # Uncomment to use PowerShell auth # token_credential = AzurePowerShellCredential() # Uncomment to use SPN auth # client_id = "your-client-id" # client_secret = "your-client-secret" # tenant_id = "your-tenant-id" # token_credential = ClientSecretCredential(client_id=client_id, client_secret=client_secret, tenant_id=tenant_id) constants.DEFAULT_API_ROOT_URL = "https://msitapi.fabric.microsoft.com" # Initialize the FabricWorkspace object with the required parameters target_workspace = FabricWorkspace( workspace_id=workspace_id, environment=environment, repository_directory=repository_directory, item_type_in_scope=item_type_in_scope, # Explicit token credential required for auth (choose one of the options above) token_credential=token_credential, ) # Uncomment to publish # Publish all items defined in item_type_in_scope # publish_all_items(target_workspace) # Uncomment to unpublish # Unpublish all items defined in scope not found in repository # unpublish_all_orphan_items(target_workspace, item_name_exclude_regex=r"^DEBUG.*") ================================================ FILE: devtools/debug_parameterization.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # The following is intended for developers of fabric-cicd to debug parameter.yml file locally against the github repo import sys from pathlib import Path import fabric_cicd.constants as constants from fabric_cicd import change_log_level from fabric_cicd._parameter._utils import validate_parameter_file root_directory = Path(__file__).resolve().parent.parent sys.path.insert(0, str(root_directory / "src")) # Uncomment to enable debug # change_log_level() # In this example, the parameter.yml file sits within the root/sample/workspace directory repository_directory = str(root_directory / "sample" / "workspace") # Explicitly define valid item types item_type_in_scope = ["DataPipeline", "Notebook", "Environment", "SemanticModel", "Report"] # Set target environment environment = "PPE" # Uncomment to use a parameter file in a different location (default location is within repository directory) # Use absolute path # parameter_file_path = str(root_directory / "sample" / "config" / "parameter.yml") # or use relative path # parameter_file_path = "../config/parameter.yml" validate_parameter_file( repository_directory=repository_directory, item_type_in_scope=item_type_in_scope, # Comment to exclude target environment in validation environment=environment, # Uncomment to use a different parameter file name within the repository directory (default name: parameter.yml) # Assign to the constant in constants.py or pass in a string directly # parameter_file_name=constants.PARAMETER_FILE_NAME, # Uncomment to use a parameter file from outside the repository (takes precedence over parameter_file_name) # parameter_file_path=parameter_file_path ) ================================================ FILE: devtools/debug_trace_deployment.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # Captures route traces from a live Fabric workspace deployment into a JSON GZIP trace file. import gzip import os import shutil import sys from pathlib import Path from azure.identity import AzureCliCredential root_directory = Path(__file__).resolve().parent.parent sys.path.insert(0, str(root_directory / "src")) import fabric_cicd def main(): """Capture HTTP trace while publishing all items to Fabric workspace.""" os.environ["FABRIC_CICD_HTTP_TRACE_ENABLED"] = "1" os.environ["FABRIC_CICD_HTTP_TRACE_FILE"] = str(root_directory / "http_trace.json") workspace_id = os.environ.get("FABRIC_WORKSPACE_ID") if not workspace_id: msg = "FABRIC_WORKSPACE_ID environment variable must be set" raise ValueError(msg) environment = "PPE" repository_directory = str(root_directory / "sample" / "workspace") item_type_in_scope = [ "DataBuildToolJob", "Dataflow", "DataPipeline", "Environment", "Eventhouse", "Eventstream", "KQLDatabase", "KQLQueryset", "Lakehouse", "MirroredDatabase", "MLExperiment", "Notebook", "Ontology", "Reflex", "Report", "SemanticModel", "SparkJobDefinition", "SQLDatabase", "VariableLibrary", "Warehouse", ] token_credential = AzureCliCredential() for flag in ["enable_shortcut_publish", "continue_on_shortcut_failure"]: fabric_cicd.append_feature_flag(flag) target_workspace = fabric_cicd.FabricWorkspace( workspace_id=workspace_id, environment=environment, repository_directory=str(repository_directory), item_type_in_scope=item_type_in_scope, token_credential=token_credential, ) fabric_cicd.publish_all_items(target_workspace) print("Publish completed successfully") # The raw JSON trace file is very large; GZIP compress it generate a compact version # that can be used by tests. The raw trace file is still left in place for # debugging purposes. # trace_file = root_directory / "http_trace.json" compressed_file = root_directory / "http_trace.json.gz" with trace_file.open("rb") as f_in, gzip.open(compressed_file, "wb") as f_out: shutil.copyfileobj(f_in, f_out) print(f"Compressed trace file to {compressed_file}") if __name__ == "__main__": main() ================================================ FILE: devtools/pypi_build_release_dev.ps1 ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. Remove-Item -Recurse -Force dist/* python -m build python -m twine upload --repository testpypi dist/* pip install --upgrade --index-url https://test.pypi.org/simple/ fabric-cicd[dev] ================================================ FILE: docs/about.md ================================================ # About ## Security {% include-markdown "../SECURITY.md" start="## Security" heading-offset=1 %} ## License {% include "../LICENSE" %} ## Get help This project uses [GitHub Issues](https://github.com/microsoft/fabric-cicd/issues) to track bugs, feature requests, and questions. Please search the existing issues before filing new issues to avoid duplicates. For new issues, file your [bug](https://github.com/microsoft/fabric-cicd/issues/new?template=1-bug.yml), [feature request](https://github.com/microsoft/fabric-cicd/issues/new?template=2-feature.yml), or [question](https://github.com/microsoft/fabric-cicd/issues/new?template=4-question.yml) as a new issue. **Found a bug or have a suggestion?** - [Raise a GitHub issue](https://github.com/microsoft/fabric-cicd/issues) - [Submit ideas to Fabric Ideas Portal](https://ideas.fabric.microsoft.com/) **Need help using the fabric-cicd library?** - For debugging information, including how to use the error log file and debug scripts, see the [Troubleshooting Guide](how_to/troubleshooting.md). - Connect with the community on [r/MicrosoftFabric](https://www.reddit.com/r/MicrosoftFabric/) - Join the [Microsoft Developer Community](https://community.fabric.microsoft.com/t5/Developer/bd-p/Developer) **Need enterprise assistance?** - Contact your Microsoft account manager or: - Open a ticket with the [Fabric Support Team](https://support.fabric.microsoft.com/) ================================================ FILE: docs/changelog.md ================================================ # Changelog ## [v1.0.0](https://pypi.org/project/fabric-cicd/1.0.0) - April 20, 2026 ### ⚠️ Breaking Change - Remove default credential fallback; explicit token credential is now required by [shirasassoon](https://github.com/shirasassoon) ([#909](https://github.com/microsoft/fabric-cicd/issues/909)) - Remove implicit authentication in Microsoft Fabric Notebook and identity logging; require explicit token credential and keyword-only arguments for `FabricWorkspace` and `deploy_with_config` by [shirasassoon](https://github.com/shirasassoon) ([#930](https://github.com/microsoft/fabric-cicd/issues/930)) ### 🆕 New Items Support - Add support for Ontology item type by [shirasassoon](https://github.com/shirasassoon) ([#796](https://github.com/microsoft/fabric-cicd/issues/796)) ### ✨ New Functionality - Improve transparency of `deploy_with_config` function in success and failure scenarios by [shirasassoon](https://github.com/shirasassoon) ([#695](https://github.com/microsoft/fabric-cicd/issues/695)) - Extend API response collection to unpublish operations by [shirasassoon](https://github.com/shirasassoon) ([#877](https://github.com/microsoft/fabric-cicd/issues/877)) - Add `get_changed_items()` utility function to detect Fabric items changed via git diff for use with selective deployment by [vipulb91](https://github.com/vipulb91) ([#865](https://github.com/microsoft/fabric-cicd/issues/865)) ### 🔧 Bug Fix - Prevent unintended GUID replacements in Variable Library item files during publish by [shirasassoon](https://github.com/shirasassoon) ([#884](https://github.com/microsoft/fabric-cicd/issues/884)) - Fix YAML content check to reject notebook and other non-YAML files during `key_value_replace` parameterization, preventing file corruption by [shirasassoon](https://github.com/shirasassoon) ([#890](https://github.com/microsoft/fabric-cicd/issues/890)) - Fix Notebook deployment failure caused by non-deterministic ordering of definition files in API payload by [shirasassoon](https://github.com/shirasassoon) ([#869](https://github.com/microsoft/fabric-cicd/issues/869)) - Ignore parameter file when not explicitly defined in config file by [aviatco](https://github.com/aviatco) ([#866](https://github.com/microsoft/fabric-cicd/issues/866)) - Add `enable_hard_delete` feature flag to bypass workspace recycle bin during unpublish by [shirasassoon](https://github.com/shirasassoon) ([#924](https://github.com/microsoft/fabric-cicd/issues/924)) - Add timeout for long-running operation polling by [shirasassoon](https://github.com/shirasassoon) ([#919](https://github.com/microsoft/fabric-cicd/issues/919)) ## [v0.3.1](https://pypi.org/project/fabric-cicd/0.3.1) - March 12, 2026 ### 🔧 Bug Fix - Fix override behavior of feature flags and constants in config deployment by [shirasassoon](https://github.com/shirasassoon) ([#872](https://github.com/microsoft/fabric-cicd/issues/872)) ## [v0.3.0](https://pypi.org/project/fabric-cicd/0.3.0) - March 09, 2026 ### ✨ New Functionality - Support selective folder deployment using inclusion list by [shirasassoon](https://github.com/shirasassoon) ([#757](https://github.com/microsoft/fabric-cicd/issues/757)) - Add FABRIC_CICD_VERSION_CHECK_DISABLED environment variable to disable version check on startup by [Ricapar](https://github.com/Ricapar) ([#811](https://github.com/microsoft/fabric-cicd/issues/811)) - Add enhanced logging configuration options via public functions by [shirasassoon](https://github.com/shirasassoon) ([#842](https://github.com/microsoft/fabric-cicd/issues/842)) - Add deployment support for Notebook items with `.ipynb` file format by [shirasassoon](https://github.com/shirasassoon) ([#850](https://github.com/microsoft/fabric-cicd/issues/850)) ### 🔧 Bug Fix - Add max workers soft cap with override and handle garbage response from Fabric Data Plane by [mdrakiburrahman](https://github.com/mdrakiburrahman) ([#827](https://github.com/microsoft/fabric-cicd/issues/827)) - Fix parameter validation failure for item names with accented characters by [shirasassoon](https://github.com/shirasassoon) ([#818](https://github.com/microsoft/fabric-cicd/issues/818)) - Exclude items with placeholder logical ID from duplicate logical ID check by [shirasassoon](https://github.com/shirasassoon) ([#843](https://github.com/microsoft/fabric-cicd/issues/843)) ### ⚡ Additional Optimizations - Add return value to deploy_with_config by [ayeshurun](https://github.com/ayeshurun) ([#851](https://github.com/microsoft/fabric-cicd/issues/851)) - Add support for Python 3.13 in the library by [shirasassoon](https://github.com/shirasassoon) ([#855](https://github.com/microsoft/fabric-cicd/issues/855)) ### 📝 Documentation Update - Remove incorrect statement on support for parameter replacements in Platform files by [shirasassoon](https://github.com/shirasassoon) ([#839](https://github.com/microsoft/fabric-cicd/issues/839)) ## [v0.2.0](https://pypi.org/project/fabric-cicd/0.2.0) - February 16, 2026 ### ✨ New Functionality - Support parallelize deployments within a given item type by [mdrakiburrahman](https://github.com/mdrakiburrahman) ([#719](https://github.com/microsoft/fabric-cicd/issues/719)) - Add a black-box REST API testing harness by [mdrakiburrahman](https://github.com/mdrakiburrahman) ([#738](https://github.com/microsoft/fabric-cicd/issues/738)) - Change header print messages to info log by [mwc360](https://github.com/mwc360) ([#771](https://github.com/microsoft/fabric-cicd/issues/771)) - Add support for semantic model binding per environment by [shirasassoon](https://github.com/shirasassoon) ([#689](https://github.com/microsoft/fabric-cicd/issues/689)) ### 🔧 Bug Fix - Remove OrgApp item type support by [shirasassoon](https://github.com/shirasassoon) ([#758](https://github.com/microsoft/fabric-cicd/issues/758)) - Improve environment-mapping behavior in optional config fields by [shirasassoon](https://github.com/shirasassoon) ([#716](https://github.com/microsoft/fabric-cicd/issues/716)) - Fix duplicate YAML key detection in parameter validation by [shirasassoon](https://github.com/shirasassoon) ([#752](https://github.com/microsoft/fabric-cicd/issues/752)) - Add caching for item attribute lookups by [MiSchroe](https://github.com/MiSchroe) ([#704](https://github.com/microsoft/fabric-cicd/issues/704)) ### ⚡ Additional Optimizations - Enable configuration-based deployment without feature flags by [shirasassoon](https://github.com/shirasassoon) ([#805](https://github.com/microsoft/fabric-cicd/issues/805)) ### 📝 Documentation Update - Fix troubleshooting docs by [shirasassoon](https://github.com/shirasassoon) ([#747](https://github.com/microsoft/fabric-cicd/issues/747)) ## [v0.1.34](https://pypi.org/project/fabric-cicd/0.1.34) - January 20, 2026 ### ✨ New Functionality - Enable dynamic replacement of SQL endpoint values from SQL Database items ([#720](https://github.com/microsoft/fabric-cicd/issues/720)) - Support Fabric Notebook Authentication ([#707](https://github.com/microsoft/fabric-cicd/issues/707)) ### 🆕 New Items Support - Onboard Spark Job Definition item type ([#115](https://github.com/microsoft/fabric-cicd/issues/115)) ### 📝 Documentation Update - Add `CONTRIBUTING.md` file to repository ([#723](https://github.com/microsoft/fabric-cicd/issues/723)) - Add comprehensive troubleshooting guide to documentation ([#705](https://github.com/microsoft/fabric-cicd/issues/705)) - Add parameterization documentation for Report items using ByConnection binding to Semantic Models ([#637](https://github.com/microsoft/fabric-cicd/issues/637)) ### ⚡ Additional Optimizations - Add debug file for local Fabric REST API testing ([#714](https://github.com/microsoft/fabric-cicd/issues/714)) ## [v0.1.33](https://pypi.org/project/fabric-cicd/0.1.33) - December 16, 2025 ### ✨ New Functionality - Add key_value_replace parameter support for YAML files ([#649](https://github.com/microsoft/fabric-cicd/issues/649)) - Support selective shortcut publishing with regex exclusion ([#624](https://github.com/microsoft/fabric-cicd/issues/624)) ### ⚡ Additional Optimizations - Add Linux development environment bootstrapping script ([#680](https://github.com/microsoft/fabric-cicd/issues/680)) - Update item types in scope to be an optional parameter in validate parameter file function ([#669](https://github.com/microsoft/fabric-cicd/issues/669)) ### 🔧 Bug Fix - Fix publish order for Notebook and Eventhouse dependent items ([#685](https://github.com/microsoft/fabric-cicd/issues/685)) - Enable parameterizing multiple connections in the same Semantic Model item ([#674](https://github.com/microsoft/fabric-cicd/issues/674)) - Fix missing description metadata in item payload for shell-only item deployments ([#672](https://github.com/microsoft/fabric-cicd/issues/672)) - Resolve API long running operation handling when publishing Environment items ([#668](https://github.com/microsoft/fabric-cicd/issues/668)) ## [v0.1.32](https://pypi.org/project/fabric-cicd/0.1.32) - December 03, 2025 ### 🔧 Bug Fix - Fix publish bug for Environment items that contain only spark settings ([#664](https://github.com/microsoft/fabric-cicd/issues/664)) ## [v0.1.31](https://pypi.org/project/fabric-cicd/0.1.31) - December 01, 2025 ### ⚠️ Breaking Change - Migrate to the latest Fabric Environment item APIs to simplify deployment and improve compatibility ([#173](https://github.com/microsoft/fabric-cicd/issues/173)) ### ✨ New Functionality - Enable dynamic replacement of Lakehouse SQL Endpoint IDs ([#616](https://github.com/microsoft/fabric-cicd/issues/616)) - Enable linking of Semantic Models to both cloud and gateway connections ([#602](https://github.com/microsoft/fabric-cicd/issues/602)) - Allow use of the dynamic replacement variables within the key_value_replace parameter ([#567](https://github.com/microsoft/fabric-cicd/issues/567)) - Add support for parameter file templates ([#499](https://github.com/microsoft/fabric-cicd/issues/499)) ### 🆕 New Items Support - Add support for the ML Experiment item type ([#600](https://github.com/microsoft/fabric-cicd/issues/600)) - Add support for the User Data Function item type ([#588](https://github.com/microsoft/fabric-cicd/issues/588)) ### 📝 Documentation Update - Update the advanced Dataflow parameterization example with the correct file_path value ([#633](https://github.com/microsoft/fabric-cicd/issues/633)) ### 🔧 Bug Fix - Fix publishing issues for KQL Database items in folders ([#657](https://github.com/microsoft/fabric-cicd/issues/657)) - Separate logic for 'items to include' feature between publish and unpublish operations ([#650](https://github.com/microsoft/fabric-cicd/issues/650)) - Fix parameterization logic to properly handle find_value regex patterns and replacements ([#639](https://github.com/microsoft/fabric-cicd/issues/639)) - Correct the publish order of Data Agent and Semantic Model items ([#628](https://github.com/microsoft/fabric-cicd/issues/628)) - Fix Lakehouse item publishing errors when shortcuts refer to the default Lakehouse ID ([#610](https://github.com/microsoft/fabric-cicd/issues/610)) ## [v0.1.30](https://pypi.org/project/fabric-cicd/0.1.30) - October 20, 2025 ### ✨ New Functionality - Add support for binding semantic models to on-premise gateways in Fabric workspaces ([#569](https://github.com/microsoft/fabric-cicd/issues/569)) ### 🆕 New Items Support - Add support for publishing and managing Data Agent items ([#556](https://github.com/microsoft/fabric-cicd/issues/556)) - Add OrgApp item type support ([#586](https://github.com/microsoft/fabric-cicd/issues/586)) ### ⚡ Additional Optimizations - Enhance cross-workspace variable support to allow referencing other attributes ([#583](https://github.com/microsoft/fabric-cicd/issues/583)) ### 🔧 Bug Fix - Fix workspace name extraction bug for non-ID attrs using ITEM_ATTR_LOOKUP ([#583](https://github.com/microsoft/fabric-cicd/issues/583)) - Fix capacity requirement check ([#593](https://github.com/microsoft/fabric-cicd/issues/593)) ## [v0.1.29](https://pypi.org/project/fabric-cicd/0.1.29) - October 01, 2025 ### ✨ New Functionality - Support dynamic replacement for cross-workspace item IDs ([#558](https://github.com/microsoft/fabric-cicd/issues/558)) - Add option to return API response for publish operations in publish_all_items ([#497](https://github.com/microsoft/fabric-cicd/issues/497)) ### 🆕 New Items Support - Onboard Apache Airflow Job item type ([#565](https://github.com/microsoft/fabric-cicd/issues/565)) - Onboard Mounted Data Factory item type ([#406](https://github.com/microsoft/fabric-cicd/issues/406)) ### 🔧 Bug Fix - Fix publish order of Eventhouses and Semantic Models ([#566](https://github.com/microsoft/fabric-cicd/issues/566)) ## [v0.1.28](https://pypi.org/project/fabric-cicd/0.1.28) - September 15, 2025 ### ✨ New Functionality - Add folder exclusion feature for publish operations ([#427](https://github.com/microsoft/fabric-cicd/issues/427)) - Expand workspace ID dynamic replacement capabilities in parameterization ([#408](https://github.com/microsoft/fabric-cicd/issues/408)) ### 🔧 Bug Fix - Fix unexpected behavior with file_path parameter filter ([#545](https://github.com/microsoft/fabric-cicd/issues/545)) - Fix unpublish exclude_regex bug in configuration file-based deployment ([#544](https://github.com/microsoft/fabric-cicd/issues/544)) ## [v0.1.27](https://pypi.org/project/fabric-cicd/0.1.27) - September 05, 2025 ### 🔧 Bug Fix - Fix trailing comma in report schema ([#534](https://github.com/microsoft/fabric-cicd/issues/534)) ## [v0.1.26](https://pypi.org/project/fabric-cicd/0.1.26) - September 05, 2025 ### ⚠️ Breaking Change - Deprecate Base API URL kwarg in Fabric Workspace ([#529](https://github.com/microsoft/fabric-cicd/issues/529)) ### ✨ New Functionality - Support Schedules parameterization ([#508](https://github.com/microsoft/fabric-cicd/issues/508)) - Support YAML configuration file-based deployment ([#470](https://github.com/microsoft/fabric-cicd/issues/470)) ### 📝 Documentation Update - Add dynamically generated Python version requirements to documentation ([#520](https://github.com/microsoft/fabric-cicd/issues/520)) ### ⚡ Additional Optimizations - Enhance pytest output to limit console verbosity ([#514](https://github.com/microsoft/fabric-cicd/issues/514)) ### 🔧 Bug Fix - Fix Report item schema handling ([#518](https://github.com/microsoft/fabric-cicd/issues/518)) - Fix deployment order to publish Mirrored Database before Lakehouse ([#482](https://github.com/microsoft/fabric-cicd/issues/482)) ## [v0.1.25](https://pypi.org/project/fabric-cicd/0.1.25) - August 19, 2025 ### ⚠️ Breaking Change - Modify the default for item_types_in_scope and add thorough validation ([#464](https://github.com/microsoft/fabric-cicd/issues/464)) ### ✨ New Functionality - Add new experimental feature flag to enable selective deployment ([#384](https://github.com/microsoft/fabric-cicd/issues/384)) - Support "ALL" environment concept in parameterization ([#320](https://github.com/microsoft/fabric-cicd/issues/320)) ### 📝 Documentation Update - Enhance Overview section in Parameterization docs ([#495](https://github.com/microsoft/fabric-cicd/issues/495)) ### ⚡ Additional Optimizations - Eliminate ACCEPTED_ITEM_TYPES_NON_UPN constant and unify with ACCEPTED_ITEM_TYPES ([#477](https://github.com/microsoft/fabric-cicd/issues/477)) - Add comprehensive GitHub Copilot instructions for effective codebase development ([#468](https://github.com/microsoft/fabric-cicd/issues/468)) ### 🔧 Bug Fix - Add feature flags and warnings for Warehouse, SQL Database, and Eventhouse unpublish operations ([#483](https://github.com/microsoft/fabric-cicd/issues/483)) - Fix code formatting inconsistencies in fabric_workspace unit test ([#474](https://github.com/microsoft/fabric-cicd/issues/474)) - Fix KeyError when deploying Reports with Semantic Model dependencies in Report-only scope case ([#278](https://github.com/microsoft/fabric-cicd/issues/278)) ## [v0.1.24](https://pypi.org/project/fabric-cicd/0.1.24) - August 04, 2025 ### ⚠️ Breaking Change - Require parameterization for Dataflow and Semantic Model references in Data Pipeline activities - Require specific parameterization for deploying a Dataflow that depends on another in the same workspace (see Parameterization docs) ### 📝 Documentation Update - Improve Parameterization documentation ([#415](https://github.com/microsoft/fabric-cicd/issues/415)) ### ⚡ Additional Optimizations - Support for Eventhouse query URI parameterization ([#414](https://github.com/microsoft/fabric-cicd/issues/414)) - Support for Warehouse SQL endpoint parameterization ([#392](https://github.com/microsoft/fabric-cicd/issues/392)) ### 🔧 Bug Fix - Fix Dataflow/Data Pipeline deployment failures caused by workspace permissions ([#419](https://github.com/microsoft/fabric-cicd/issues/419)) - Prevent duplicate logical ID issue in Report and Semantic Model deployment ([#405](https://github.com/microsoft/fabric-cicd/issues/405)) - Fix deployment of items without assigned capacity ([#402](https://github.com/microsoft/fabric-cicd/issues/402)) ## [v0.1.23](https://pypi.org/project/fabric-cicd/0.1.23) - July 08, 2025 ### ✨ New Functionality - New functionalities for GitHub Copilot Agent and PR-to-Issue linking ### 📝 Documentation Update - Fix formatting and examples in the How to and Examples pages ### 🔧 Bug Fix - Fix issue with lakehouse shortcuts publishing ([#379](https://github.com/microsoft/fabric-cicd/issues/379)) - Add validation for empty logical IDs to prevent deployment corruption ([#86](https://github.com/microsoft/fabric-cicd/issues/86)) - Fix SQL provision print statement ([#329](https://github.com/microsoft/fabric-cicd/issues/329)) - Rename the error code for reserved item name per updated Microsoft Fabric API ([#388](https://github.com/microsoft/fabric-cicd/issues/388)) - Fix lakehouse exclude_regex to exclude shortcut publishing ([#385](https://github.com/microsoft/fabric-cicd/issues/385)) - Remove max retry limit to handle large deployments ([#299](https://github.com/microsoft/fabric-cicd/issues/299)) ## [v0.1.22](https://pypi.org/project/fabric-cicd/0.1.22) - June 25, 2025 ### 🆕 New Items Support - Onboard API for GraphQL item type ([#287](https://github.com/microsoft/fabric-cicd/issues/287)) ### 🔧 Bug Fix - Fix Fabric API call error during dataflow publish ([#352](https://github.com/microsoft/fabric-cicd/issues/352)) ### ⚡ Additional Optimizations - Expanded test coverage to handle folder edge cases ([#358](https://github.com/microsoft/fabric-cicd/issues/358)) ## [v0.1.21](https://pypi.org/project/fabric-cicd/0.1.21) - June 18, 2025 ### 🔧 Bug Fix - Fix bug with workspace ID replacement in JSON files for pipeline deployments ([#345](https://github.com/microsoft/fabric-cicd/issues/345)) ### ⚡ Additional Optimizations - Increased max retry for Warehouses and Dataflows ## [v0.1.20](https://pypi.org/project/fabric-cicd/0.1.20) - June 12, 2025 ### ✨ New Functionality - Parameterization support for find_value regex and replace_value variables ([#326](https://github.com/microsoft/fabric-cicd/issues/326)) ### 🆕 New Items Support - Onboard KQL Dashboard item type ([#329](https://github.com/microsoft/fabric-cicd/issues/329)) - Onboard Dataflow Gen2 item type ([#111](https://github.com/microsoft/fabric-cicd/issues/111)) ### 🔧 Bug Fix - Fix bug with deploying environment libraries with special chars ([#336](https://github.com/microsoft/fabric-cicd/issues/336)) ### ⚡ Additional Optimizations - Improved test coverage for subfolder creation/modification ([#211](https://github.com/microsoft/fabric-cicd/issues/211)) ## [v0.1.19](https://pypi.org/project/fabric-cicd/0.1.19) - May 21, 2025 ### 🆕 New Items Support - Onboard SQL Database item type (shell-only deployment) ([#301](https://github.com/microsoft/fabric-cicd/issues/301)) - Onboard Warehouse item type (shell-only deployment) ([#204](https://github.com/microsoft/fabric-cicd/issues/204)) ### 🔧 Bug Fix - Fix bug with unpublish workspace folders ([#273](https://github.com/microsoft/fabric-cicd/issues/273)) ## [v0.1.18](https://pypi.org/project/fabric-cicd/0.1.18) - May 14, 2025 ### 🔧 Bug Fix - Fix bug with check environment publish state ([#295](https://github.com/microsoft/fabric-cicd/issues/295)) ## [v0.1.17](https://pypi.org/project/fabric-cicd/0.1.17) - May 13, 2025 ### ⚠️ Breaking Change - Deprecate old parameter file structure ([#283](https://github.com/microsoft/fabric-cicd/issues/283)) ### 🆕 New Items Support - Onboard CopyJob item type ([#122](https://github.com/microsoft/fabric-cicd/issues/122)) - Onboard Eventstream item type ([#170](https://github.com/microsoft/fabric-cicd/issues/170)) - Onboard Eventhouse/KQL Database item type ([#169](https://github.com/microsoft/fabric-cicd/issues/169)) - Onboard Data Activator item type ([#291](https://github.com/microsoft/fabric-cicd/issues/291)) - Onboard KQL Queryset item type ([#292](https://github.com/microsoft/fabric-cicd/issues/292)) ### 🔧 Bug Fix - Fix post publish operations for skipped items ([#277](https://github.com/microsoft/fabric-cicd/issues/277)) ### ⚡ Additional Optimizations - New function `key_value_replace` for key-based replacement operations in JSON and YAML ### 📝 Documentation Update - Add publish regex example to demonstrate how to use the `publish_all_items` with regex for excluding item names ## [v0.1.16](https://pypi.org/project/fabric-cicd/0.1.16) - April 25, 2025 ### 🔧 Bug Fix - Fix bug with folder deployment to root ([#255](https://github.com/microsoft/fabric-cicd/issues/255)) ### ⚡ Additional Optimizations - Add Workspace Name in FabricWorkspaceObject ([#200](https://github.com/microsoft/fabric-cicd/issues/200)) - New function to check SQL endpoint provision status ([#226](https://github.com/microsoft/fabric-cicd/issues/226)) ### 📝 Documentation Update - Updated Authentication docs + menu sort order ## [v0.1.15](https://pypi.org/project/fabric-cicd/0.1.15) - April 21, 2025 ### 🔧 Bug Fix - Fix folders moving with every publish ([#236](https://github.com/microsoft/fabric-cicd/issues/236)) ### ⚡ Additional Optimizations - Introduce parallel deployments to reduce publish times ([#237](https://github.com/microsoft/fabric-cicd/issues/237)) - Improvements to check version logic ### 📝 Documentation Update - Updated Examples section in docs ## [v0.1.14](https://pypi.org/project/fabric-cicd/0.1.14) - April 09, 2025 ### ✨ New Functionality - Optimized & beautified terminal output - Added changelog to output of old version check ### 🔧 Bug Fix - Fix workspace folder deployments in root folder ([#221](https://github.com/microsoft/fabric-cicd/issues/221)) - Fix unpublish of workspace folders without publish ([#222](https://github.com/microsoft/fabric-cicd/issues/222)) ### ⚡ Additional Optimizations - Removed Colorama and Colorlog Dependency ## [v0.1.13](https://pypi.org/project/fabric-cicd/0.1.13) - April 07, 2025 ### ✨ New Functionality - Added support for Lakehouse Shortcuts - New `enable_environment_variable_replacement` feature flag ([#160](https://github.com/microsoft/fabric-cicd/issues/160)) ### 🆕 New Items Support - Onboard Workspace Folders ([#81](https://github.com/microsoft/fabric-cicd/issues/81)) - Onboard Variable Library item type ([#206](https://github.com/microsoft/fabric-cicd/issues/206)) ### ⚡ Additional Optimizations - User-agent now available in API headers ([#207](https://github.com/microsoft/fabric-cicd/issues/207)) - Fixed error log typo in fabric_endpoint ### 🔧 Bug Fix - Fix break with invalid optional parameters ([#192](https://github.com/microsoft/fabric-cicd/issues/192)) - Fix bug where all workspace ids were not being replaced by parameterization ([#186](https://github.com/microsoft/fabric-cicd/issues/186)) ## [v0.1.12](https://pypi.org/project/fabric-cicd/0.1.12) - March 27, 2025 ### 🔧 Bug Fix - Fix constant overwrite failures ([#190](https://github.com/microsoft/fabric-cicd/issues/190)) - Fix bug where all workspace ids were not being replaced ([#186](https://github.com/microsoft/fabric-cicd/issues/186)) - Fix type hints for older versions of Python ([#156](https://github.com/microsoft/fabric-cicd/issues/156)) - Fix accepted item types constant in pre-build ## [v0.1.11](https://pypi.org/project/fabric-cicd/0.1.11) - March 25, 2025 ### ⚠️ Breaking Change - Parameterization refactor introducing a new parameter file structure and parameter file validation functionality ([#113](https://github.com/microsoft/fabric-cicd/issues/113)) ### ✨ New Functionality - Support regex for publish exclusion ([#121](https://github.com/microsoft/fabric-cicd/issues/121)) - Override max retries via constants ([#146](https://github.com/microsoft/fabric-cicd/issues/146)) ### 📝 Documentation Update - Update to [parameterization](https://microsoft.github.io/fabric-cicd/latest/how_to/parameterization/) docs ## [v0.1.10](https://pypi.org/project/fabric-cicd/0.1.10) - March 19, 2025 ### ✨ New Functionality - DataPipeline SPN Support ([#133](https://github.com/microsoft/fabric-cicd/issues/133)) ### 🔧 Bug Fix - Workspace ID replacement in data pipelines ([#164](https://github.com/microsoft/fabric-cicd/issues/164)) ### 📝 Documentation Update - Sample for passing in arguments from Azure DevOps Pipelines ## [v0.1.9](https://pypi.org/project/fabric-cicd/0.1.9) - March 11, 2025 ### 🆕 New Items Support - Support for Mirrored Database item type ([#145](https://github.com/microsoft/fabric-cicd/issues/145)) ### ⚡ Additional Optimizations - Increase reserved name wait time ([#135](https://github.com/microsoft/fabric-cicd/issues/135)) ## [v0.1.8](https://pypi.org/project/fabric-cicd/0.1.8) - March 04, 2025 ### 🔧 Bug Fix - Handle null byPath object in report definition file ([#143](https://github.com/microsoft/fabric-cicd/issues/143)) - Support relative directories ([#136](https://github.com/microsoft/fabric-cicd/issues/136)) ([#132](https://github.com/microsoft/fabric-cicd/issues/132)) - Increase special character support ([#134](https://github.com/microsoft/fabric-cicd/issues/134)) ### ⚡ Additional Optimizations - Changelog now available with version check ([#127](https://github.com/microsoft/fabric-cicd/issues/127)) ## [v0.1.7](https://pypi.org/project/fabric-cicd/0.1.7) - February 26, 2025 ### 🔧 Bug Fix - Fix special character support in files ([#129](https://github.com/microsoft/fabric-cicd/issues/129)) ## [v0.1.6](https://pypi.org/project/fabric-cicd/0.1.6) - February 24, 2025 ### 🆕 New Items Support - Onboard Lakehouse item type ([#116](https://github.com/microsoft/fabric-cicd/issues/116)) ### 📝 Documentation Update - Update example docs ([#25](https://github.com/microsoft/fabric-cicd/issues/25)) - Update find_replace docs ([#110](https://github.com/microsoft/fabric-cicd/issues/110)) ### ⚡ Additional Optimizations - Standardized docstrings to Google format - Onboard file objects ([#46](https://github.com/microsoft/fabric-cicd/issues/46)) - Leverage UpdateDefinition Flag ([#28](https://github.com/microsoft/fabric-cicd/issues/28)) - Convert repo and workspace dictionaries ([#45](https://github.com/microsoft/fabric-cicd/issues/45)) ## [v0.1.5](https://pypi.org/project/fabric-cicd/0.1.5) - February 18, 2025 ### 🔧 Bug Fix - Fix Environment Failure without Public Library ([#103](https://github.com/microsoft/fabric-cicd/issues/103)) ### ⚡ Additional Optimizations - Introduces pytest check for PRs ([#100](https://github.com/microsoft/fabric-cicd/issues/100)) ## [v0.1.4](https://pypi.org/project/fabric-cicd/0.1.4) - February 12, 2025 ### ✨ New Functionality - Support Feature Flagging ([#96](https://github.com/microsoft/fabric-cicd/issues/96)) ### 🔧 Bug Fix - Fix Image support in report deployment ([#88](https://github.com/microsoft/fabric-cicd/issues/88)) - Fix Broken README link ([#92](https://github.com/microsoft/fabric-cicd/issues/92)) ### ⚡ Additional Optimizations - Workspace ID replacement improved - Increased error handling in activate script - Onboard pytest and coverage - Improvements to nested dictionaries ([#37](https://github.com/microsoft/fabric-cicd/issues/37)) - Support Python Installed From Windows Store ([#87](https://github.com/microsoft/fabric-cicd/issues/87)) ## [v0.1.3](https://pypi.org/project/fabric-cicd/0.1.3) - January 29, 2025 ### ✨ New Functionality - Add PyPI check version to encourage version bumps ([#75](https://github.com/microsoft/fabric-cicd/issues/75)) ### 🔧 Bug Fix - Fix Semantic model initial publish results in None Url error ([#61](https://github.com/microsoft/fabric-cicd/issues/61)) - Fix Integer parsed as float failing in handle_retry for <3.12 python ([#63](https://github.com/microsoft/fabric-cicd/issues/63)) - Fix Default item types fail to unpublish ([#76](https://github.com/microsoft/fabric-cicd/issues/76)) - Fix Items in subfolders are skipped ([#77](https://github.com/microsoft/fabric-cicd/issues/77)) ### 📝 Documentation Update - Update documentation & examples ## [v0.1.2](https://pypi.org/project/fabric-cicd/0.1.2) - January 27, 2025 ### ✨ New Functionality - Introduces max retry and backoff for long running / throttled calls ([#27](https://github.com/microsoft/fabric-cicd/issues/27)) ### 🔧 Bug Fix - Fix Environment publish uses arbitrary wait time ([#50](https://github.com/microsoft/fabric-cicd/issues/50)) - Fix Environment publish doesn't wait for success ([#56](https://github.com/microsoft/fabric-cicd/issues/56)) - Fix Long running operation steps out early for notebook publish ([#58](https://github.com/microsoft/fabric-cicd/issues/58)) ## [v0.1.1](https://pypi.org/project/fabric-cicd/0.1.1) - January 23, 2025 ### 🔧 Bug Fix - Fix Environment stuck in publish ([#51](https://github.com/microsoft/fabric-cicd/issues/51)) ## [v0.1.0](https://pypi.org/project/fabric-cicd/0.1.0) - January 23, 2025 ### ✨ New Functionality - Initial public preview release - Supports Notebook, Pipeline, Semantic Model, Report, and Environment deployments - Supports User and System Identity authentication - Released to PyPi - Onboarded to Github Pages ================================================ FILE: docs/code_reference.md ================================================ # Code Reference ::: fabric_cicd ================================================ FILE: docs/config/overrides/main.html ================================================ {% extends "base.html" %} {% block announce %}

ℹ️ This library is open source. Please raise issues & feature requests as they arise.

{% endblock %} ================================================ FILE: docs/config/pre-build/section_toc.py ================================================ from pathlib import Path import re import unicodedata import yaml def slugify(title): """ Generate an anchor slug from a heading title, matching MkDocs/Python-Markdown toc behavior. Strips markdown formatting (backticks, escape chars), lowercases, replaces spaces with hyphens, and removes characters that aren't alphanumeric, hyphens, or underscores. """ # Remove markdown escape backslashes (e.g. \_ALL\_ -> _ALL_) slug = title.replace("\\", "") # Remove backticks (inline code markers) slug = slug.replace("`", "") # Normalize unicode slug = unicodedata.normalize("NFKD", slug) # Lowercase slug = slug.lower() # Replace spaces with hyphens slug = slug.replace(" ", "-") # Remove characters that aren't alphanumeric, hyphens, or underscores slug = re.sub(r"[^\w\-]", "", slug) # Strip leading/trailing hyphens slug = slug.strip("-") return slug def get_section_order(nav, current_dir_str): """ Recursively find the order of markdown files in the current directory as defined in nav. """ order = [] for item in nav: if isinstance(item, dict): for key, value in item.items(): if isinstance(value, list): # Recurse into subsections order += get_section_order(value, current_dir_str) elif isinstance(value, str): path = Path(value) if str(path.parent).replace("\\", "/") == current_dir_str: order.append(path.name) elif isinstance(item, str): path = Path(item) if str(path.parent).replace("\\", "/") == current_dir_str: order.append(path.name) return order def on_page_markdown(markdown, page, config, files): if "\n" in markdown: start_index = markdown.index("\n") + len("\n") end_index = markdown.index("\n") current_page_path = Path(page.file.abs_src_path) current_page_dir = current_page_path.parent # Compute docs root (where mkdocs.yml lives) mkdocs_yml = Path(config['config_file_path']) docs_root = mkdocs_yml.parent # Get current dir relative to docs root, as string current_dir_rel = str(current_page_dir.name) # Load mkdocs.yml and extract nav order with mkdocs_yml.open("r", encoding="utf-8") as f: mkdocs_config = yaml.safe_load(f) nav = mkdocs_config.get('nav', []) section_order = get_section_order(nav, current_dir_rel) toc = [] for md_name in section_order: md_file = current_page_dir / md_name if not md_file.exists() or md_file == current_page_path: continue with md_file.open("r", encoding="utf-8") as f: content = f.read() content_no_code = re.sub(r"```.*?```", "", content, flags=re.DOTALL) headers = re.findall(r"^(#{1,6})\s+(.*)", content_no_code, re.MULTILINE) seen_slugs = {} for header in headers: level = len(header[0]) title = header[1] base_anchor = slugify(title) # MkDocs appends _1, _2, etc. for duplicate heading IDs count = seen_slugs.get(base_anchor, 0) seen_slugs[base_anchor] = count + 1 anchor = base_anchor if count == 0 else f"{base_anchor}_{count}" toc.append(f"{' ' * level}- [{title}]({md_file.name}#{anchor})") toc_content = "\n".join(toc) new_markdown = markdown[:start_index] + toc_content + markdown[end_index:] return new_markdown return markdown ================================================ FILE: docs/config/pre-build/update_item_types.py ================================================ import sys from pathlib import Path root_directory = Path(__file__).resolve().parent.parent.parent.parent sys.path.insert(0, str(root_directory / "src")) import fabric_cicd.constants as constants def on_page_markdown(markdown, **kwargs): if "\n" in markdown: start_index = markdown.index("\n") + len("\n") end_index = markdown.index("\n") supported_item_types = constants.ACCEPTED_ITEM_TYPES markdown_content = "\n".join([f"- {item}" for item in supported_item_types]) new_markdown = markdown[:start_index] + markdown_content + markdown[end_index:] return new_markdown return markdown ================================================ FILE: docs/config/pre-build/update_python_version.py ================================================ import re try: import tomllib # Python 3.11+ except ImportError: import toml as tomllib # Fallback for older Python versions from pathlib import Path def on_page_markdown(markdown, **kwargs): """ Replace Python version placeholders with versions from pyproject.toml """ if "" in markdown or "" in markdown: # Get pyproject.toml path (4 levels up from this file) root_directory = Path(__file__).resolve().parent.parent.parent.parent pyproject_path = root_directory / "pyproject.toml" try: # Load pyproject.toml with open(pyproject_path, 'rb') as f: try: pyproject_data = tomllib.load(f) except AttributeError: # Fallback for older toml library f.seek(0) content = f.read().decode('utf-8') pyproject_data = tomllib.loads(content) # Extract requires-python requires_python = pyproject_data.get('project', {}).get('requires-python', '') # Extract min and max versions min_version = "3.9" # fallback max_version = "3.12" # fallback if requires_python: min_match = re.search(r'>=(\d+\.\d+)', requires_python) max_match = re.search(r'<(\d+\.\d+)', requires_python) if min_match: min_version = min_match.group(1) if max_match: # Convert exclusive max to inclusive (e.g., <3.13 means up to 3.12) max_parts = max_match.group(1).split('.') max_major = int(max_parts[0]) max_minor = int(max_parts[1]) if max_minor > 0: max_version = f"{max_major}.{max_minor - 1}" except Exception: # Use fallback values min_version = "3.9" max_version = "3.13" # Replace placeholders markdown = markdown.replace("", min_version) markdown = markdown.replace("", max_version) return markdown ================================================ FILE: docs/config/stylesheets/extra.css ================================================ [data-md-color-scheme="fabric"] { --md-primary-fg-color: #117865; --md-primary-fg-color--light: #e3f7ef; --md-primary-fg-color--dark: #012826; --md-typeset-a-color: #012826; --md-accent-fg-color: #012826; --md-default-fg-color--light: #333333; --md-typeset-a-color: #117865; --md-primary-bg-color: #ffffff; } /* Remove opaqueness to top nav */ .md-tabs__link, .md-tabs__link:hover, .md-tabs__item--active { opacity: 1; } /* Add line under active top nav item */ .md-tabs__item--active { opacity: 1; box-shadow: inset 0 -6px 0 0 var(--md-primary-fg-color), inset 0 -8px 0 0 var(--md-primary-bg-color); } /* Add underline to links */ .md-content a { text-decoration: underline; color: var(--md-primary-fg-color); } /* Keep underline on hover */ .md-content a:hover { text-decoration: underline; color: var(--md-primary-fg-color--dark); } .md-typeset .tabbed-labels > label > [href]:first-child { color: var(--md-typeset-a-color); text-decoration: inherit; } .js .md-typeset .tabbed-labels:before { background: var(--md-typeset-a-color); } /*Code inside of tables are of white background*/ .md-typeset table code { background-color: var(--md-primary-bg-color); } /*Code inside of tables are colored when hovered*/ .md-typeset table tr:hover code { background-color: inherit; } /*Default header color*/ .md-content h1, .md-content h2, .md-content h3 { color: var(--md-accent-fg-color); font-weight: 500; /* Slight bold */ margin: 0 0 1em; } /*Add a border line to all H2*/ .md-content h2 { padding-top: 1em; border-top: 1px solid #ddd; } /*Custom class for subheaders under h2*/ .md-h2-subheader { font-size: smaller; color: inherit; } /*Custom class for non header h3*/ .md-h3-nonanchor { color: var(--md-accent-fg-color); font-weight: 500; /* Slight bold */ margin: 0 0 1em; font-size: 1.25em; } .md-h4-nonanchor { color: var(--md-accent-fg-color); font-weight: 700; letter-spacing: -0.01em; margin: 1em 0; } /*Remove all margins from the

element containing .md-h2-subheader*/ p:has(.md-h2-subheader) { margin: 0; } /*Remove bottom margin from h2 so it's closer to subheader*/ .md-content h2:has(+ p .md-h2-subheader) { margin-bottom: 0; } /* Override default font size for typeset */ .md-typeset { font-size: 0.64rem; } .md-typeset h1 { font-size: 1.25rem; } .md-typeset h2 { font-size: 1rem; } .md-nav--lifted .md-nav__item { font-size: 0.7rem; } /* Markdown image resizing */ .md-content .md-typeset img { max-width: 40rem; width: 100%; height: auto; /* Maintain aspect ratio */ } ================================================ FILE: docs/example/authentication.md ================================================ # Authentication Examples The following are the most common authentication flows for fabric-cicd. However, because fabric-cicd supports any [TokenCredential](https://learn.microsoft.com/en-us/dotnet/api/azure.core.tokencredential), there are multiple authentication methods available beyond the ones described here. These examples provide starting points that should be adapted for your specific environment and security requirements. > **⚠️ NOTICE:** Due to security best practices, the **Default Credential** (`DefaultAzureCredential` fallback) and **implicit Fabric Notebook authentication** (without a `token_credential` parameter) methods are no longer supported. `token_credential` is now a required parameter. **Notes:** - Fabric Notebook users must provide an explicit `token_credential`. See [Fabric Notebook Authentication](#fabric-notebook-authentication) for options. - Avoid hardcoding credentials. Use environment variables or secret management services. SPN + Secret auth can also be achieved via `az login --service-principal` or `Connect-AzAccount -ServicePrincipal` in the CLI/PowerShell flows below. ## CLI Credential This approach utilizes the CLI credential flow, meaning it only refers to the authentication established with `az login`. This is agnostic of the executing user; it can be UPN, SPN, Managed Identity, etc. Whatever is used to log in will be used. === "Local" ```python '''Log in with Azure CLI (az login) prior to execution''' from pathlib import Path from azure.identity import AzureCliCredential from fabric_cicd import FabricWorkspace, publish_all_items, unpublish_all_orphan_items # Assumes your script is one level down from root root_directory = Path(__file__).resolve().parent # Sample values for FabricWorkspace parameters workspace_id = "your-workspace-id" environment = "your-environment" repository_directory = str(root_directory / "your-workspace-directory") item_type_in_scope = ["Notebook", "DataPipeline", "Environment"] # Use Azure CLI credential to authenticate token_credential = AzureCliCredential() # Initialize the FabricWorkspace object with the required parameters target_workspace = FabricWorkspace( workspace_id=workspace_id, environment=environment, repository_directory=repository_directory, item_type_in_scope=item_type_in_scope, token_credential=token_credential, ) # Publish all items defined in item_type_in_scope publish_all_items(target_workspace) # Unpublish all items defined in item_type_in_scope not found in repository unpublish_all_orphan_items(target_workspace) ``` === "Azure DevOps" ```python ''' Log in with Azure CLI (az login) prior to execution OR (Preferred) Use Az CLI ADO Tasks with a Service Connection ''' import sys import os from pathlib import Path from azure.identity import AzureCliCredential from fabric_cicd import FabricWorkspace, publish_all_items, unpublish_all_orphan_items, change_log_level # Force unbuffered output like `python -u` sys.stdout.reconfigure(line_buffering=True, write_through=True) sys.stderr.reconfigure(line_buffering=True, write_through=True) # Enable debugging if defined in Azure DevOps pipeline if os.getenv("SYSTEM_DEBUG", "false").lower() == "true": change_log_level("DEBUG") # Assumes your script is one level down from root root_directory = Path(__file__).resolve().parent # Sample values for FabricWorkspace parameters workspace_id = "your-workspace-id" environment = "your-environment" repository_directory = str(root_directory / "your-workspace-directory") item_type_in_scope = ["Notebook", "DataPipeline", "Environment"] # Use Azure CLI credential to authenticate token_credential = AzureCliCredential() # Initialize the FabricWorkspace object with the required parameters target_workspace = FabricWorkspace( workspace_id=workspace_id, environment=environment, repository_directory=repository_directory, item_type_in_scope=item_type_in_scope, token_credential=token_credential, ) # Publish all items defined in item_type_in_scope publish_all_items(target_workspace) # Unpublish all items defined in item_type_in_scope not found in repository unpublish_all_orphan_items(target_workspace) ``` === "GitHub" ```python ''' Log in with Azure CLI (az login) prior to execution Requires: azure/login workflow step in GitHub Actions ''' import os from pathlib import Path from azure.identity import AzureCliCredential from fabric_cicd import FabricWorkspace, publish_all_items, unpublish_all_orphan_items # GitHub Actions sets GITHUB_WORKSPACE automatically root_directory = Path(os.getenv("GITHUB_WORKSPACE", ".")).resolve() # Sample values for FabricWorkspace parameters workspace_id = os.getenv("WORKSPACE_ID") environment = os.getenv("ENVIRONMENT", "PROD") repository_directory = str(root_directory / "your-workspace-directory") item_type_in_scope = ["Notebook", "DataPipeline", "Environment"] # Use Azure CLI credential (assumes 'az login' in workflow step) token_credential = AzureCliCredential() # Initialize the FabricWorkspace object with the required parameters target_workspace = FabricWorkspace( workspace_id=workspace_id, environment=environment, repository_directory=repository_directory, item_type_in_scope=item_type_in_scope, token_credential=token_credential, ) # Publish all items defined in item_type_in_scope publish_all_items(target_workspace) # Unpublish all items defined in item_type_in_scope not found in repository unpublish_all_orphan_items(target_workspace) ``` ## AZ PowerShell Credential This approach utilizes the AZ PowerShell credential flow, meaning it only refers to the authentication established with `Connect-AzAccount`. This is agnostic of the executing user; it can be UPN, SPN, Managed Identity, etc. Whatever is used to log in will be used. === "Local" ```python '''Log in with Azure PowerShell (Connect-AzAccount) prior to execution''' from pathlib import Path from azure.identity import AzurePowerShellCredential from fabric_cicd import FabricWorkspace, publish_all_items, unpublish_all_orphan_items # Assumes your script is one level down from root root_directory = Path(__file__).resolve().parent # Sample values for FabricWorkspace parameters workspace_id = "your-workspace-id" environment = "your-environment" repository_directory = str(root_directory / "your-workspace-directory") item_type_in_scope = ["Notebook", "DataPipeline", "Environment"] # Use Azure PowerShell credential to authenticate token_credential = AzurePowerShellCredential() # Initialize the FabricWorkspace object with the required parameters target_workspace = FabricWorkspace( workspace_id=workspace_id, environment=environment, repository_directory=repository_directory, item_type_in_scope=item_type_in_scope, token_credential=token_credential, ) # Publish all items defined in item_type_in_scope publish_all_items(target_workspace) # Unpublish all items defined in item_type_in_scope not found in repository unpublish_all_orphan_items(target_workspace) ``` === "Azure DevOps" ```python ''' Log in with Azure PowerShell (Connect-AzAccount) prior to execution OR (Preferred) Use AzPowerShell ADO Tasks with a Service Connection ''' import sys import os from pathlib import Path from azure.identity import AzurePowerShellCredential from fabric_cicd import FabricWorkspace, publish_all_items, unpublish_all_orphan_items, change_log_level # Force unbuffered output like `python -u` sys.stdout.reconfigure(line_buffering=True, write_through=True) sys.stderr.reconfigure(line_buffering=True, write_through=True) # Enable debugging if defined in Azure DevOps pipeline if os.getenv("SYSTEM_DEBUG", "false").lower() == "true": change_log_level("DEBUG") # Assumes your script is one level down from root root_directory = Path(__file__).resolve().parent # Sample values for FabricWorkspace parameters workspace_id = "your-workspace-id" environment = "your-environment" repository_directory = str(root_directory / "your-workspace-directory") item_type_in_scope = ["Notebook", "DataPipeline", "Environment"] # Use Azure PowerShell credential to authenticate token_credential = AzurePowerShellCredential() # Initialize the FabricWorkspace object with the required parameters target_workspace = FabricWorkspace( workspace_id=workspace_id, environment=environment, repository_directory=repository_directory, item_type_in_scope=item_type_in_scope, token_credential=token_credential, ) # Publish all items defined in item_type_in_scope publish_all_items(target_workspace) # Unpublish all items defined in item_type_in_scope not found in repository unpublish_all_orphan_items(target_workspace) ``` === "GitHub" ```python ''' Log in with Azure PowerShell (Connect-AzAccount) prior to execution Requires: azure/powershell workflow step in GitHub Actions ''' import os from pathlib import Path from azure.identity import AzurePowerShellCredential from fabric_cicd import FabricWorkspace, publish_all_items, unpublish_all_orphan_items # GitHub Actions sets GITHUB_WORKSPACE automatically root_directory = Path(os.getenv("GITHUB_WORKSPACE", ".")).resolve() # Sample values for FabricWorkspace parameters workspace_id = os.getenv("WORKSPACE_ID") environment = os.getenv("ENVIRONMENT", "PROD") repository_directory = str(root_directory / "your-workspace-directory") item_type_in_scope = ["Notebook", "DataPipeline", "Environment"] # Use Azure PowerShell credential (assumes 'Connect-AzAccount' in workflow step) token_credential = AzurePowerShellCredential() # Initialize the FabricWorkspace object with the required parameters target_workspace = FabricWorkspace( workspace_id=workspace_id, environment=environment, repository_directory=repository_directory, item_type_in_scope=item_type_in_scope, token_credential=token_credential, ) # Publish all items defined in item_type_in_scope publish_all_items(target_workspace) # Unpublish all items defined in item_type_in_scope not found in repository unpublish_all_orphan_items(target_workspace) ``` ## Managed Identity Credential This approach uses Azure Managed Identity, eliminating the need to manage secrets. Managed identities provide an automatically managed identity in Azure AD for applications to use when connecting to resources. === "Azure DevOps" ```python ''' Running on Azure DevOps self-hosted agents with system-assigned managed identity OR Azure DevOps agents hosted on Azure VMs with managed identity ''' import sys import os from pathlib import Path from azure.identity import ManagedIdentityCredential from fabric_cicd import FabricWorkspace, publish_all_items, unpublish_all_orphan_items, change_log_level # Force unbuffered output like `python -u` sys.stdout.reconfigure(line_buffering=True, write_through=True) sys.stderr.reconfigure(line_buffering=True, write_through=True) # Enable debugging if defined in Azure DevOps pipeline if os.getenv("SYSTEM_DEBUG", "false").lower() == "true": change_log_level("DEBUG") # Assumes your script is one level down from root root_directory = Path(__file__).resolve().parent # Sample values for FabricWorkspace parameters workspace_id = "your-workspace-id" environment = "your-environment" repository_directory = str(root_directory / "your-workspace-directory") item_type_in_scope = ["Notebook", "DataPipeline", "Environment"] # Use system-assigned managed identity token_credential = ManagedIdentityCredential() # Initialize the FabricWorkspace object with the required parameters target_workspace = FabricWorkspace( workspace_id=workspace_id, environment=environment, repository_directory=repository_directory, item_type_in_scope=item_type_in_scope, token_credential=token_credential, ) # Publish all items defined in item_type_in_scope publish_all_items(target_workspace) # Unpublish all items defined in item_type_in_scope not found in repository unpublish_all_orphan_items(target_workspace) ``` === "GitHub" ```python ''' Running on GitHub self-hosted runners with system-assigned managed identity OR GitHub Actions hosted on Azure VMs with managed identity ''' import os from pathlib import Path from azure.identity import ManagedIdentityCredential from fabric_cicd import FabricWorkspace, publish_all_items, unpublish_all_orphan_items # GitHub Actions sets GITHUB_WORKSPACE automatically root_directory = Path(os.getenv("GITHUB_WORKSPACE", ".")).resolve() # Sample values for FabricWorkspace parameters workspace_id = os.getenv("WORKSPACE_ID") environment = os.getenv("ENVIRONMENT", "PROD") repository_directory = str(root_directory / "your-workspace-directory") item_type_in_scope = ["Notebook", "DataPipeline", "Environment"] # Use system-assigned managed identity token_credential = ManagedIdentityCredential() # Initialize the FabricWorkspace object with the required parameters target_workspace = FabricWorkspace( workspace_id=workspace_id, environment=environment, repository_directory=repository_directory, item_type_in_scope=item_type_in_scope, token_credential=token_credential, ) # Publish all items defined in item_type_in_scope publish_all_items(target_workspace) # Unpublish all items defined in item_type_in_scope not found in repository unpublish_all_orphan_items(target_workspace) ``` ## Fabric Notebook Authentication When running fabric-cicd within Microsoft Fabric Notebooks, an explicit `token_credential` parameter is required, consistent with all other environments. The simplest approach uses `notebookutils.credentials.getToken()` to create a credential from the current user session. Alternatively, you can use a service principal for specific identity requirements. === "Session Credential" ```python ''' Use the Fabric Notebook session identity via notebookutils Most common pattern: clone repository and deploy from within notebook ''' import time import tempfile import subprocess import os from azure.core.credentials import AccessToken, TokenCredential from fabric_cicd import FabricWorkspace, publish_all_items, unpublish_all_orphan_items # Define a credential that wraps the Fabric session token class FabricNotebookCredential(TokenCredential): def get_token(self, *scopes, **kwargs): token = notebookutils.credentials.getToken("pbi") return AccessToken(token, int(time.time()) + 3600) # 1 hour # Sample configuration values workspace_id = "your-workspace-id" environment = "your-environment" repo_url = "https://github.com/your-org/your-repo.git" repo_ref = "main" workspace_directory = "your-workspace-directory" # Use context manager for automatic cleanup (even on exceptions) with tempfile.TemporaryDirectory(prefix="cloned_repo_") as temp_dir: print(f"Created temporary directory: {temp_dir}") # Clone the repository print(f"Cloning {repo_url} (ref: {repo_ref})...") result = subprocess.run( ["git", "clone", "--branch", repo_ref, "--single-branch", repo_url, temp_dir], capture_output=True, text=True ) if result.returncode != 0: raise Exception(f"Git clone failed: {result.stderr}") workspace_root = os.path.join(temp_dir, workspace_directory) # Deploy workspace items from cloned repository item_type_in_scope = ["Notebook", "DataPipeline", "Environment"] # Initialize FabricWorkspace with explicit session credential target_workspace = FabricWorkspace( workspace_id=workspace_id, environment=environment, repository_directory=workspace_root, item_type_in_scope=item_type_in_scope, token_credential=FabricNotebookCredential(), ) # Publish all items defined in item_type_in_scope publish_all_items(target_workspace) # Unpublish all items defined in item_type_in_scope not found in repository unpublish_all_orphan_items(target_workspace) # Directory automatically cleaned up here print("Cleaned up temporary directory") ``` === "SPN Credential" ```python ''' Use a service principal for specific identity requirements Retrieve secrets from Azure Key Vault using notebookutils ''' import tempfile import subprocess import os from azure.identity import ClientSecretCredential from fabric_cicd import FabricWorkspace, publish_all_items, unpublish_all_orphan_items # Sample configuration values workspace_id = "your-workspace-id" environment = "your-environment" repo_url = "https://github.com/your-org/your-repo.git" repo_ref = "main" workspace_directory = "your-workspace-directory" # Retrieve secrets from Azure Key Vault using notebookutils key_vault_url = "https://your-keyvault.vault.azure.net/" client_id = notebookutils.credentials.getSecret(key_vault_url, "client-id") client_secret = notebookutils.credentials.getSecret(key_vault_url, "client-secret") tenant_id = notebookutils.credentials.getSecret(key_vault_url, "tenant-id") token_credential = ClientSecretCredential( client_id=client_id, client_secret=client_secret, tenant_id=tenant_id ) # Use context manager for automatic cleanup (even on exceptions) with tempfile.TemporaryDirectory(prefix="cloned_repo_") as temp_dir: print(f"Created temporary directory: {temp_dir}") # Clone the repository print(f"Cloning {repo_url} (ref: {repo_ref})...") result = subprocess.run( ["git", "clone", "--branch", repo_ref, "--single-branch", repo_url, temp_dir], capture_output=True, text=True ) if result.returncode != 0: raise Exception(f"Git clone failed: {result.stderr}") workspace_root = os.path.join(temp_dir, workspace_directory) # Deploy workspace items from cloned repository item_type_in_scope = ["Notebook", "DataPipeline", "Environment"] # Initialize with explicit SPN credential target_workspace = FabricWorkspace( workspace_id=workspace_id, environment=environment, repository_directory=workspace_root, item_type_in_scope=item_type_in_scope, token_credential=token_credential, ) # Publish all items defined in item_type_in_scope publish_all_items(target_workspace) # Unpublish all items defined in item_type_in_scope not found in repository unpublish_all_orphan_items(target_workspace) # Directory automatically cleaned up here print("Cleaned up temporary directory") ``` ================================================ FILE: docs/example/deployment_variable.md ================================================ # Deployment Variable Examples A key concept in CI/CD is defining environment-specific deployment variables. The following are examples of how to inject variables from outside of the python script to handle values that are environment-specific, or common across other tooling. These examples provide starting points that should be adapted for your specific environment and security requirements. > **⚠️ NOTICE**: Due to security best practices, the **Default Credential** (`DefaultAzureCredential` fallback) and **implicit Fabric Notebook authentication** (without a `token_credential` parameter) methods are no longer supported. `token_credential` is now a required parameter. **Note:** All examples below use the `AzureCliCredential` token for demonstration purposes. You can substitute this with other explicit credential methods based on your environment. ## Branch Based Leverage the following when you have specific values that you need to define per branch you are deploying from. === "Local" ```python '''Determines variables based on locally checked out branch''' from pathlib import Path import git # Depends on pip install gitpython from azure.identity import AzureCliCredential from fabric_cicd import FabricWorkspace, publish_all_items, unpublish_all_orphan_items # Use Azure CLI credential to authenticate token_credential = AzureCliCredential() # Assumes your script is one level down from root root_directory = Path(__file__).resolve().parent repo = git.Repo(root_directory) repo.remotes.origin.pull() branch = repo.active_branch.name # The defined environment values should match the names found in the parameter.yml file if branch == "dev": workspace_id = "dev-workspace-id" environment = "DEV" elif branch == "main": workspace_id = "prod-workspace-id" environment = "PROD" else: raise ValueError("Invalid branch to deploy from") # Sample values for FabricWorkspace parameters repository_directory = str(root_directory / "your-workspace-directory") item_type_in_scope = ["Notebook", "DataPipeline", "Environment"] # Initialize the FabricWorkspace object with the required parameters target_workspace = FabricWorkspace( workspace_id=workspace_id, environment=environment, repository_directory=repository_directory, item_type_in_scope=item_type_in_scope, token_credential=token_credential, # or any other TokenCredential ) # Publish all items defined in item_type_in_scope publish_all_items(target_workspace) # Unpublish all items defined in item_type_in_scope not found in repository unpublish_all_orphan_items(target_workspace) ``` === "Azure DevOps" ```python ''' Determines variables based on the branch that originated the build Uses Azure CLI credential with service connection ''' import sys import os from pathlib import Path from azure.identity import AzureCliCredential from fabric_cicd import FabricWorkspace, publish_all_items, unpublish_all_orphan_items, change_log_level # Force unbuffered output like `python -u` sys.stdout.reconfigure(line_buffering=True, write_through=True) sys.stderr.reconfigure(line_buffering=True, write_through=True) # Enable debugging if defined in Azure DevOps pipeline if os.getenv("SYSTEM_DEBUG", "false").lower() == "true": change_log_level("DEBUG") # Use Azure CLI credential to authenticate token_credential = AzureCliCredential() # Assumes your script is one level down from root root_directory = Path(__file__).resolve().parent branch = os.getenv("BUILD_SOURCEBRANCHNAME") # The defined environment values should match the names found in the parameter.yml file if branch == "dev": workspace_id = "dev-workspace-id" environment = "DEV" elif branch == "main": workspace_id = "prod-workspace-id" environment = "PROD" else: raise ValueError("Invalid branch to deploy from") # Sample values for FabricWorkspace parameters repository_directory = str(root_directory / "your-workspace-directory") item_type_in_scope = ["Notebook", "DataPipeline", "Environment"] # Initialize the FabricWorkspace object with the required parameters target_workspace = FabricWorkspace( workspace_id=workspace_id, environment=environment, repository_directory=repository_directory, item_type_in_scope=item_type_in_scope, token_credential=token_credential, # or any other TokenCredential ) # Publish all items defined in item_type_in_scope publish_all_items(target_workspace) # Unpublish all items defined in item_type_in_scope not found in repository unpublish_all_orphan_items(target_workspace) ``` === "GitHub" ```python ''' Determines variables based on the branch that originated the build Uses Azure CLI credential (requires az login in GitHub Actions workflow) ''' import os from pathlib import Path from azure.identity import AzureCliCredential from fabric_cicd import FabricWorkspace, publish_all_items, unpublish_all_orphan_items # Use Azure CLI credential to authenticate token_credential = AzureCliCredential() # GitHub Actions sets GITHUB_WORKSPACE automatically root_directory = Path(os.getenv("GITHUB_WORKSPACE", ".")).resolve() # Get branch from GitHub environment variable branch = os.getenv("GITHUB_REF_NAME") # The defined environment values should match the names found in the parameter.yml file if branch == "dev": workspace_id = "dev-workspace-id" environment = "DEV" elif branch == "main": workspace_id = "prod-workspace-id" environment = "PROD" else: raise ValueError("Invalid branch to deploy from") # Sample values for FabricWorkspace parameters repository_directory = str(root_directory / "your-workspace-directory") item_type_in_scope = ["Notebook", "DataPipeline", "Environment"] # Initialize the FabricWorkspace object with the required parameters target_workspace = FabricWorkspace( workspace_id=workspace_id, environment=environment, repository_directory=repository_directory, item_type_in_scope=item_type_in_scope, token_credential=token_credential, # or any other TokenCredential ) # Publish all items defined in item_type_in_scope publish_all_items(target_workspace) # Unpublish all items defined in item_type_in_scope not found in repository unpublish_all_orphan_items(target_workspace) ``` ## Passed Arguments Leverage the following when you want to pass in variables outside of the python script. This is most common for scenarios where you want to use one py script, but have multiple deployments. === "Local" ```python '''Accepts parameters passed into Python during execution''' import argparse from azure.identity import AzureCliCredential from fabric_cicd import FabricWorkspace, publish_all_items, unpublish_all_orphan_items # Use Azure CLI credential to authenticate token_credential = AzureCliCredential() # Accept parsed arguments parser = argparse.ArgumentParser(description='Process deployment arguments.') parser.add_argument('--workspace_id', type=str, required=True) parser.add_argument('--environment', type=str) parser.add_argument('--repository_directory', type=str, required=True) parser.add_argument('--items_in_scope', type=str) args = parser.parse_args() # Sample values for FabricWorkspace parameters workspace_id = args.workspace_id environment = args.environment repository_directory = args.repository_directory item_type_in_scope = args.items_in_scope.split(",") if args.items_in_scope else ["Notebook", "DataPipeline", "Environment"] # Initialize the FabricWorkspace object with the required parameters target_workspace = FabricWorkspace( workspace_id=workspace_id, environment=environment, repository_directory=repository_directory, item_type_in_scope=item_type_in_scope, token_credential=token_credential, # or any other TokenCredential ) # Publish all items defined in item_type_in_scope publish_all_items(target_workspace) # Unpublish all items defined in item_type_in_scope not found in repository unpublish_all_orphan_items(target_workspace) ``` === "Azure DevOps" ```python ''' Accepts parameters passed into Python during execution Uses Azure CLI credential with service connection ''' import sys import os import argparse from pathlib import Path from azure.identity import AzureCliCredential from fabric_cicd import FabricWorkspace, publish_all_items, unpublish_all_orphan_items, change_log_level # Force unbuffered output like `python -u` sys.stdout.reconfigure(line_buffering=True, write_through=True) sys.stderr.reconfigure(line_buffering=True, write_through=True) # Enable debugging if defined in Azure DevOps pipeline if os.getenv("SYSTEM_DEBUG", "false").lower() == "true": change_log_level("DEBUG") # Use Azure CLI credential to authenticate token_credential = AzureCliCredential() # Accept parsed arguments parser = argparse.ArgumentParser(description='Process deployment arguments.') parser.add_argument('--workspace_id', type=str, required=True) parser.add_argument('--environment', type=str) parser.add_argument('--repository_directory', type=str, required=True) parser.add_argument('--items_in_scope', type=str) args = parser.parse_args() # Sample values for FabricWorkspace parameters workspace_id = args.workspace_id environment = args.environment repository_directory = args.repository_directory item_type_in_scope = args.items_in_scope.split(",") if args.items_in_scope else ["Notebook", "DataPipeline", "Environment"] # Initialize the FabricWorkspace object with the required parameters target_workspace = FabricWorkspace( workspace_id=workspace_id, environment=environment, repository_directory=repository_directory, item_type_in_scope=item_type_in_scope, token_credential=token_credential, # or any other TokenCredential ) # Publish all items defined in item_type_in_scope publish_all_items(target_workspace) # Unpublish all items defined in item_type_in_scope not found in repository unpublish_all_orphan_items(target_workspace) ``` === "GitHub" ```python '''Unconfirmed example at this time, however, the Azure DevOps example is a good starting point''' ``` ================================================ FILE: docs/example/index.md ================================================ # How To Welcome to the Examples section! Here you will find all necessary code samples to leverage fabric-cicd. If there is any missing information, raise a [documentation issue](https://github.com/microsoft/fabric-cicd/issues/new?template=3-documentation.yml) on GitHub. ## Contents ================================================ FILE: docs/example/release_pipeline.md ================================================ # Release Pipeline Examples The following are some common examples of how to deploy from tooling like Azure DevOps and GitHub. Note that this is not an exhaustive list, nor is it a recommendation to not use a proper Build/Release stage. These are simplified to show the potential. ## Azure CLI This approach uses the Azure CLI Credential Flow. An explicit credential method is required. This avoids ambiguity when multiple identities are present in the build VM. === "Azure DevOps" ```yml trigger: branches: include: - dev - main stages: - stage: Build_Release jobs: - job: Build pool: vmImage: windows-latest steps: - checkout: self - task: UsePythonVersion@0 inputs: versionSpec: '3.12' addToPath: true - script: | pip install fabric-cicd displayName: 'Install fabric-cicd' - task: AzureCLI@2 displayName: "Deploy Fabric Workspace" inputs: azureSubscription: "your-service-connection" scriptType: "ps" scriptLocation: "inlineScript" inlineScript: | python -u $(System.DefaultWorkingDirectory)/.deploy/fabric_workspace.py ``` === "GitHub" ```yaml # Unconfirmed example at this time. The Azure DevOps example is a good starting point. ``` ## Azure PowerShell This approach uses the Azure PowerShell Credential Flow. An explicit credential method is required. This avoids ambiguity when multiple identities are present in the build VM. === "Azure DevOps" ```yml trigger: branches: include: - dev - main stages: - stage: Build_Release jobs: - job: Build pool: vmImage: windows-latest steps: - checkout: self - task: UsePythonVersion@0 inputs: versionSpec: '3.12' addToPath: true - script: | pip install fabric-cicd displayName: 'Install fabric-cicd' - task: AzurePowerShell@5 displayName: "Deploy Fabric Workspace" inputs: azureSubscription: "your-service-connection" scriptType: "InlineScript" scriptLocation: "inlineScript" pwsh: true Inline: | python -u $(System.DefaultWorkingDirectory)/.deploy/fabric_workspace.py ``` === "GitHub" ```yaml # Unconfirmed example at this time. The Azure DevOps example is a good starting point. ``` ## Variable Groups This approach is best suited for the Passed Arguments example found in the Deployment Variable Examples, in combination with a `ClientSecretCredential` as shown in the [Authentication Examples](authentication.md). The goal is to define values within the pipeline (or outside the pipeline in Azure DevOps variable groups) and inject them into the python script. Note this also doesn't take a dependency on PowerShell for those organizations or scenarios where PowerShell is not allowed. === "Azure DevOps" ```yml trigger: branches: include: - dev - main parameters: - name: items_in_scope displayName: Enter Fabric items to be deployed type: string default: '["Notebook","DataPipeline","Environment"]' variables: - group: Fabric_Deployment_Group_KeyVault # Linked to Azure Key Vault and contains tenant id, SPN client id, and SPN secret - group: Fabric_Deployment_Group # Contains workspace_name and repository directory name stages: - stage: Build_Release jobs: - job: Build pool: vmImage: windows-latest steps: - checkout: self - task: UsePythonVersion@0 inputs: versionSpec: '3.12' addToPath: true - script: | pip install fabric-cicd displayName: 'Install fabric-cicd' - task: PythonScript@0 inputs: scriptSource: 'filePath' scriptPath: '.deploy/fabric_workspace.py' arguments: >- --spn_client_id $(client_id) # from Fabric_Deployment_Group_KeyVault --spn_client_secret $(client_secret) # from Fabric_Deployment_Group_KeyVault --tenant_id $(tenant_id) # from Fabric_Deployment_Group_KeyVault --workspace_id $(workspace_id) # from Fabric_Deployment_Group --environment $(environment_name) # from Fabric_Deployment_Group --repository_directory $(repository_directory) # from Fabric_Deployment_Group --item_types_in_scope ${{ parameters.items_in_scope }} ``` === "GitHub" ```yaml # Unconfirmed example at this time. The Azure DevOps example is a good starting point. ``` ================================================ FILE: docs/how_to/config_deployment.md ================================================ # Configuration Deployment ## Overview Configuration-based deployment provides an alternative way to manage the deployment of Fabric items across multiple environments. Instead of using the traditional approach of defining a workspace object with various parameters and then running the publish/unpublish functions, this approach centralizes all deployment settings in a single YAML configuration file and simplifies the deployment into one function call. Configuration file location (supports any location in the Git repository): ``` C:/dev/workspace /HelloWorld.Notebook ... /GoodbyeWorld.Notebook ... /config.yml ``` Basic example of configuration-based deployment: > **Note:** All parameters except `config_file_path` must be passed as keyword arguments to `deploy_with_config()`. ```python from fabric_cicd import deploy_with_config from azure.identity import AzureCliCredential # Deploy using a config file deploy_with_config( config_file_path="C:/dev/workspace/config.yml", # required token_credential=AzureCliCredential(), # required environment="dev" ) ``` Raise a [feature request](https://github.com/microsoft/fabric-cicd/issues/new?template=2-feature.yml) for additional capabilities or a [bug report](https://github.com/microsoft/fabric-cicd/issues/new?template=1-bug.yml) for issues. ## Configuration File Setup The configuration file includes several sections with configurable settings for different aspects of the deployment process. > **Note:** Configuration values can be specified in two ways: as a single value (applied to any target environment provided) or as an environment mapping. Both approaches can be used within the same configuration file — for example, using environment mappings for workspace IDs while keeping a single value for repository directory. ### Core Settings The `core` section is **required** as it defines the fundamental settings for the deployment, most importantly the **target workspace** and **repository directory**. Other optional settings can be configured within the `core` section, including **item types in scope** and **parameter**. ```yaml core: # Only one workspace identifier field is required workspace: workspace_id: # Required - path to the directory containing Fabric items repository_directory: # Optional - specific item types to include in deployment item_types_in_scope: - - - # Optional - path to parameter file parameter: ``` With environment mapping: ```yaml core: # Only one workspace identifier field is required workspace: : : workspace_id: : : # Required - path to the directory containing Fabric items repository_directory: : : # Optional - specific item types to include in deployment item_types_in_scope: : - - : - - # Optional - path to parameter file parameter: : : ``` Required Fields: - **Workspace Identifier:** - Workspace ID takes precedence over workspace name when both are provided. - `workspace_id` must be a valid string GUID. - **Repository Directory Path:** - Supports relative or absolute path. - Relative path must be relative to the `config.yml` file location. Optional Fields: - **Item Types in Scope:** - If `item_types_in_scope` is not specified, all item types will be included by default. - Item types must be provided as a list; use `-` or `[]` notation. - Only accepts supported item types. - **Parameter Path:** - Supports relative or absolute path. - Relative path must be relative to the `config.yml` file location. ### Publish Settings The `publish` section is optional and controls item publishing behavior. If this section is omitted entirely, publishing will run with **default behavior — all items published, no exclusions.** It includes various optional settings to enable/disable publishing operations or selectively publish items. > **Note:** `folder_exclude_regex` and `folder_path_to_include` are mutually exclusive — providing both for the same environment will result in a validation error. For detailed information about folder and item filtering behavior, see [Selective Deployment Features](optional_feature.md#selective-deployment-features). ```yaml publish: # Optional - pattern to exclude items from publishing (no feature flag required) exclude_regex: # Optional - pattern to exclude specific folder paths with items from publishing (requires feature flags) folder_exclude_regex: # Optional - specific folder paths with items to publish (requires feature flags) folder_path_to_include: - - - # publish items found in nested folder - subfolder_3 # Optional - specific items to publish (requires feature flags) items_to_include: - - # Optional - pattern to exclude Lakehouse shortcuts from publishing (requires feature flags) shortcut_exclude_regex: # Optional - control publishing by environment skip: ``` With environment mapping: ```yaml publish: # Optional - pattern to exclude items from publishing exclude_regex: : : # Optional - pattern to exclude specific folder paths with items from publishing (requires feature flags) folder_exclude_regex: : : # Optional - specific folder paths with items to publish (requires feature flags) folder_path_to_include: : - - : - # Optional - specific items to publish (requires feature flags) items_to_include: : - - : - - # Optional - pattern to exclude Lakehouse shortcuts from publishing (requires feature flags) shortcut_exclude_regex: : : # Optional - control publishing by environment skip: : : ``` ### Unpublish Settings The `unpublish` section is optional and controls item unpublishing behavior. If this section is omitted entirely, unpublishing will run with **default behavior — all orphan items unpublished, no exclusions.** It includes various optional settings to enable/disable unpublishing or selectively unpublish items. ```yaml unpublish: # Optional - pattern to exclude items from unpublishing (no feature flag required) exclude_regex: # Optional - specific items to unpublish (requires feature flags) items_to_include: - - # Optional - control unpublishing by environment skip: ``` With environment mapping: ```yaml unpublish: # Optional - pattern to exclude items from unpublishing exclude_regex: : : # Optional - specific items to unpublish (requires feature flags) items_to_include: : - - : - - # Optional - control unpublishing by environment skip: : : ``` > **Warning:** While selective deployment is supported in fabric-cicd, it is not recommended due to potential issues with dependency management. ### Features Setting The `features` section is optional and allows you to set a list of specific feature flags. ```yaml features: - - ``` With environment mapping: ```yaml features: : - - : - - ``` ### Constants Setting The `constants` section is optional and allows you to override supported library constants. ```yaml constants: CONSTANT_NAME: ``` With environment mapping: ```yaml constants: CONSTANT_NAME: : : ``` ## Environment-Specific Values All configuration fields support environment-specific values using a mapping format: ```yaml core: workspace_id: dev: "dev-workspace-id" test: "test-workspace-id" prod: "prod-workspace-id" ``` ### Required vs Optional Fields Fields are categorized as **required** or **optional**, which affects how missing environment values are handled when environment is passed into `deploy_with_config()`: > **Note:** When the `publish` or `unpublish` sections are omitted entirely, both operations run by default. To skip either operation, explicitly set `skip: true` for that section. | Field | Required | Environment Missing Behavior | | --------------------------------------- | -------- | ------------------------------- | | `core.workspace_id` or `core.workspace` | ✅ | Validation error | | `core.repository_directory` | ✅ | Validation error | | `core.item_types_in_scope` | ❌ | Warning logged, setting skipped | | `core.parameter` | ❌ | Warning logged, setting skipped | | `publish.exclude_regex` | ❌ | Debug logged, setting skipped | | `publish.folder_exclude_regex` | ❌ | Debug logged, setting skipped | | `publish.shortcut_exclude_regex` | ❌ | Debug logged, setting skipped | | `publish.folder_path_to_include` | ❌ | Debug logged, setting skipped | | `publish.items_to_include` | ❌ | Debug logged, setting skipped | | `publish.skip` | ❌ | Defaults to `False` | | `unpublish.exclude_regex` | ❌ | Debug logged, setting skipped | | `unpublish.items_to_include` | ❌ | Debug logged, setting skipped | | `unpublish.skip` | ❌ | Defaults to `False` | | `features` | ❌ | Warning logged, setting skipped | | `constants` | ❌ | Warning logged, setting skipped | ### Selective Environment Configuration Optional fields allow you to apply settings to specific environments without affecting others. This is useful when you want different behavior per environment: ```yaml core: workspace_id: dev: "dev-workspace-id" test: "test-workspace-id" prod: "prod-workspace-id" repository_directory: "./workspace" # Same for all environments publish: # Only exclude legacy folders in prod environment folder_exclude_regex: prod: "^/legacy_.*" # dev and test not specified - no folder exclusion applied # Skip publish in dev, run in test and prod skip: dev: true # test and prod default to false ``` In this example: - Deploying to `dev`: No folder exclusion applied, `skip` = `true` - Deploying to `test`: No folder exclusion applied, `skip` = `false` - Deploying to `prod`: `folder_exclude_regex` = `"^/legacy_.*"`, `skip` = `false` ### Logging Behavior When an optional field uses environment mapping and does not include the target environment: - **Important optional fields** (`item_types_in_scope`, `parameter`): A **warning** is logged to alert users that the setting is being skipped. - **Other optional fields**: A **debug** message is logged, visible only when debug logging is enabled. Example log output when deploying to `prod` with the configuration above: ``` [Debug] - No value for 'folder_exclude_regex' in environment 'prod'. Available environments: ['dev']. This setting will be skipped. ``` To enable debug logging: ```python from fabric_cicd import change_log_level change_log_level() ``` ## Sample `config.yml` File ```yaml core: workspace: dev: "Fabric-Dev-Engineering" test: "Fabric-Test-Engineering" prod: "Fabric-Prod-Engineering" workspace_id: dev: "8b6e2c7a-4c1f-4e3a-9b2e-7d8f2e1a6c3b" test: "2f4b9e8d-1a7c-4d3e-b8e2-5c9f7a2d4e1b" prod: "7c3e1f8b-2d4a-4b9e-8f2c-1a6c3b7d8e2f" repository_directory: "." # relative path item_types_in_scope: - Notebook - DataPipeline - Environment - Lakehouse parameter: "parameter.yml" # relative path publish: # Don't publish items matching this pattern (no feature flag required) exclude_regex: "^DONT_DEPLOY.*" # Use folder_exclude_regex OR folder_path_to_include, not both for the same environment folder_exclude_regex: dev: "^/DONT_DEPLOY_FOLDER" folder_path_to_include: prod: - "/DEPLOY_FOLDER" - "/DEPLOY_FOLDER/DEPLOY_NESTED_FOLDER" items_to_include: - "Hello World.Notebook" - "Run Hello World.DataPipeline" shortcut_exclude_regex: test: "^temp_.*" skip: dev: true test: false prod: false unpublish: # Don't unpublish items matching this pattern (no feature flag required) exclude_regex: "^DEBUG.*" skip: dev: false test: false prod: true features: - enable_shortcut_publish - enable_experimental_features - enable_items_to_include - enable_exclude_folder - enable_include_folder - enable_shortcut_exclude constants: DEFAULT_API_ROOT_URL: "https://api.fabric.microsoft.com" ``` ## Configuration File Deployment ### Basic Usage ```python from fabric_cicd import deploy_with_config from azure.identity import AzureCliCredential # Deploy using a config file deploy_with_config( config_file_path="path/to/config.yml", # required token_credential=AzureCliCredential(), # required environment="dev" # optional (recommended) ) ``` ### Custom Authentication ```python from fabric_cicd import deploy_with_config from azure.identity import ClientSecretCredential # Create a credential credential = ClientSecretCredential( tenant_id="your-tenant-id", client_id="your-client-id", client_secret="your-client-secret" ) # Deploy with custom credential deploy_with_config( config_file_path="path/to/config.yml", token_credential=credential, environment="prod" ) ``` ### Configuration Override The `config_override` parameter in `deploy_with_config()` allows you to dynamically modify configuration values at runtime without changing the base configuration file. This is particularly useful for debugging or making temporary deployment adjustments. ```python from fabric_cicd import deploy_with_config from azure.identity import AzureCliCredential config_override_dict = { "core": { "item_types_in_scope": ["Notebook", "DataPipeline"] }, "publish": { "skip": { "dev": False } } } # Deploy with configuration override deploy_with_config( config_file_path="path/to/config.yml", token_credential=AzureCliCredential(), environment="dev", config_override=config_override_dict ) ``` **Important Considerations:** - **Caution:** Exercise caution when overriding configuration values for _production_ environments. - **Support:** Configuration overrides are supported for all sections and settings in the configuration file. - **Rules:** - Existing values can be overridden for any field in the configuration. - New values can only be added for optional fields that aren't present in the original configuration. - Required fields must exist in the original configuration in order to override. ## Troubleshooting Guide The configuration file undergoes validation prior to reaching the deployment phase. Here are some common issues that may occur: 1. **File Not Found:** Ensure the configuration file path is correct and accessible (must be an absolute path). 2. **Invalid YAML:** Check YAML syntax for errors (indentation, missing quotes, etc.). 3. **Missing Required Fields:** Ensure the `core` section is present and contains the required fields (workspace identifier, repository directory path). 4. **Path Resolution Errors:** Relative paths are resolved relative to the `config.yml` file location. Check that path inputs are valid and accessible. 5. **Environment Not Found:** The `environment` parameter must match one of the environment keys (e.g., "dev", "test", "prod") used in the configuration mappings for required fields (`workspace`/`workspace_id`, `repository_directory`). ================================================ FILE: docs/how_to/getting_started.md ================================================ # Getting Started ## Installation To install fabric-cicd, run: ```bash pip install fabric-cicd ``` ## Authentication > **⚠️ NOTICE**: Due to security best practices, the **Default Credential** (`DefaultAzureCredential` fallback) and **implicit Fabric Notebook authentication** (without a `token_credential` parameter) methods are no longer supported. `token_credential` is now a required parameter. - You must provide your own credential object that aligns with the `TokenCredential` class (from [azure.identity](https://learn.microsoft.com/en-us/python/api/azure-identity/azure.identity?view=azure-python)). For more details, see the [TokenCredential](https://learn.microsoft.com/en-us/python/api/azure-core/azure.core.credentials.tokencredential?view=azure-python) documentation. - When running in Fabric Notebook runtime, provide an explicit credential. See Authentication examples for details. **Recommended Authentication Methods:** - For local development: `AzureCliCredential` or `AzurePowerShellCredential` (user authentication) - For CI/CD pipelines: `AzureCliCredential`/`AzurePowerShellCredential` (platform authentication), `ClientSecretCredential` (service principal), or `ManagedIdentityCredential` (self-hosted agents) **Basic Example:** ```python from azure.identity import AzureCliCredential from fabric_cicd import FabricWorkspace token_credential = AzureCliCredential() workspace = FabricWorkspace( workspace_id="your-workspace-id", environment="your-target-environment", repository_directory="your-repository-directory", item_type_in_scope=["Notebook", "DataPipeline", "Environment"], token_credential=token_credential, # or any other TokenCredential ) ``` See the [Authentication Examples](../example/authentication.md) for specific implementation patterns. ## Directory Structure This library deploys from a directory containing files and directories committed via the Fabric Source Control UI. Ensure the `repository_directory` includes only these committed items, with the exception of the `parameter.yml` file. ``` / /. ... /. ... / /. ... /. ... /parameter.yml ``` ## GIT Flow The flow pictured below is the hero scenario for this library and is the recommendation if you're just starting out. - `Deployed` branches are not connected to workspaces via [GIT Sync](https://learn.microsoft.com/en-us/fabric/cicd/git-integration/git-get-started?tabs=azure-devops%2CAzure%2Ccommit-to-git#connect-a-workspace-to-a-git-repo) - `Feature` branches are connected to workspaces via [GIT Sync](https://learn.microsoft.com/en-us/fabric/cicd/git-integration/git-get-started?tabs=azure-devops%2CAzure%2Ccommit-to-git#connect-a-workspace-to-a-git-repo) - `Deployed` workspaces are only updated through script-based deployments, such as through the fabric-cicd library - `Feature` branches are created from the default branch, merged back into the default `Deployed` branch, and cherry picked into the upper `Deployed` branches - Each deployment is a full deployment and does not consider commit diffs ![GIT Flow](../config/assets/git_flow.png) ================================================ FILE: docs/how_to/index.md ================================================ # How To Welcome to the How To section! Here you will find all necessary information to leverage fabric-cicd. If there is any missing information, raise a [documentation issue](https://github.com/microsoft/fabric-cicd/issues/new?template=3-documentation.yml) on GitHub. ## Contents ================================================ FILE: docs/how_to/item_types.md ================================================ # Item Types ## Activator - **Parameterization:** - The `find_replace` section in the `parameter.yml` file is not applied. - **Initial deployment** may not reflect streaming data immediately. - **Reflex** is the item name in source control. Source control may not support all activators/reflexes, as not all sources are compatible. ## Apache Airflow Job - **Parameterization:** - The referenced items in DAG files will always point to the original item unless parameterized in the `find_replace` section of the `parameter.yml` file. - **Connections** are not source controlled and must be created manually. - See known CI/CD limitations [here](https://learn.microsoft.com/en-us/fabric/data-factory/cicd-apache-airflow-jobs#known-limitations). ## API for GraphQL - **Parameterization:** - The source will always point to the source in the original workspace unless parameterized in the `find_replace` section of the `parameter.yml` file. - It is recommended to use the supported variables in `find_replace` for dynamic replacement of the source workspace and item IDs. - If you are using a connection and expect it to change between different environments, then it needs to be parameterized in the `parameter.yml` file. - When using the **Saved Credential** method to connect to data sources, developers must have access to the Saved Credential information in order to successfully deploy GraphQL item. - Changes made to the original API query are not source controlled. You will need to manually update the query in the GraphQL item's query editor within the target workspace. - Only user authentication is currently supported for GraphQL items that source data from the SQL Analytics Endpoint. ## Copy Job - **Parameterization:** - Connections will always point to the original data source unless parameterized in the `find_replace` section of the `parameter.yml` file. - **Initial deployment** requires manual configuration of the connection after deployment. ## Dataflow - **Parameterization:** - Source/destination items (e.g., Dataflow, Lakehouse, Warehouse) will always reference the original item unless parameterized in the `find_replace` section of the `parameter.yml` file. - The recommended approach for re-pointing source/destination items that exist in the _same_ workspace as the Dataflow is to use **`replace_value` variables** in the `find_replace` parameter along with a `find_value` regex (literal string works too). For more guidance, see [Parameterization -> Dataflows](parameterization.md#dataflows). - **Important** for Dataflows that reference another Dataflow in the _same_ workspace, ONLY parameterize the referenced **dataflowId** in the `mashup.pq` file (workspaceId re-pointing is handled automatically in this approach) using the following `replace_value`: `$items.Dataflow..id`. This ensures proper dependency resolution and ordered publishing. - **Initial deployment** will require a manual publish and refresh of the dataflow. After re-pointing dataflows during deployment, temporary errors may appear but should resolve after refreshing and allowing time for processing (especially when a dataflow sources from another dataflow in the target workspace). - `fabric ci-cd` automatically manages ordered deployment of dataflows that source from other dataflows in the same workspace. - **Connections** are not source controlled and require manual creation. - If you use connections that differ between environments, parameterize them in the `parameter.yml` file. ## Data Agent - **Parameterization:** - Data source items (e.g., Lakehouse, Warehouse, Semantic Model) will always reference the original item unless parameterized in the `find_replace` section of the `parameter.yml` file. ## Data Build Tool Job - **Parameterization:** - Connection and profile references will always point to the original values unless parameterized in the `find_replace` section of the `parameter.yml` file. - **Initial deployment:** - Validate connection/profile settings in the target workspace, especially when promoting between test and prod workspaces. ## Data Pipeline - **Parameterization:** - Activities connected to items that exist in a different workspace will always point to the original item unless parameterized in the `find_replace` section of the `parameter.yml` file. - Activities connected to items within the same workspace are automatically re-pointed to the new item in the target workspace. **Note: Activities referencing items that don't use _logical ID_ and _default workspace ID_ (such as Refresh Dataflow and Refresh Semantic Model) require parameterization.** - **Connections** are not source controlled and must be created manually. - If you are using connections and expect them to change between different environments, then those need to be parameterized in the `parameter.yml` file. - The **executing identity** of the deployment must have access to the connections, or the deployment will fail. ## Environment - **Parameterization:** - Environments attached to custom spark pools attach to the default starter pool unless parameterized in the `spark_pools` section of the `parameter.yml` file. - The `find_replace` section in the `parameter.yml` file is not applied to Environments. - **Resources** are not source controlled and will not be deployed. - Environments with libraries will have **high initial publish times** (sometimes 20+ minutes). ## Eventhouse - **Parameterization:** - The `find_replace` section in the `parameter.yml` file is not applied. - The `exclude_path` variable is required when deploying an **Eventhouse** that is attached to a **KQL Database** (common scenario). - There may be significant _differences_ in the streaming data between the source eventhouse and the deployed eventhouse. - **Unpublish** is disabled by default, enable with feature flag `enable_eventhouse_unpublish`. ## Eventstream - **Parameterization:** - Destinations connected to items that exist in a different workspace will always point to the original item unless parameterized in the `find_replace` section of the `parameter.yml` file. - Destinations connected to items within the same workspace are re-pointed to the new item in the target workspace. - **Initial deployment** requires waiting for the table to populate in the destination lakehouse if a lakehouse destination is present in the eventstream. ## KQL Database - **Parameterization:** - The `find_replace` section in the `parameter.yml` file is not applied. - In Fabric, a KQL database is not a standalone item. However, during deployment, it is treated as such. Its source control files are located within a `.children` folder under the directory of the attached eventhouse. - Data in KQL database tables is not source controlled and may not consistently appear in the database UI after deployment. Some tables may be empty post-deployment. ## KQL Queryset - **Parameterization:** - KQL querysets attached to KQL databases always point to the original KQL database unless parameterized in the `find_replace` section of the `parameter.yml` file. - The **cluster/query URI** of the KQL database must be present in the KQL queryset JSON for rebinding. If the KQL queryset is attached to a KQL database within the same workspace, the cluster URI value is empty and needs to be re-added. `fabric ci-cd` handles this automatically. - KQL querysets can still exist after the KQL database source has been deleted. However, errors will reflect in the KQL queryset. ## Lakehouse - **Parameterization:** - The `find_replace` section in the `parameter.yml` file is not applied. - **Shortcut** publish is disabled by default (for now), enable with feature flag `enable_shortcut_publish`. - **Schemas are not deployed** unless the schema has a shortcut present. - **Unpublish** is disabled by default, enable with feature flag `enable_lakehouse_unpublish`. ## Mirrored Database - **Parameterization:** - Connections will always point to the original source database unless parameterized in the `find_replace` section of the `parameter.yml` file. - **Initial deployment** for Azure SQL Database or Azure SQL Managed Instance requires manual granting of System Assigned Managed Identity (SAMI) Read and Write permission to the mirrored database for replication to be successful after deployment. ref -> ([Prerequisites](https://learn.microsoft.com/en-us/fabric/database/mirrored-database/mirrored-database-rest-api#create-mirrored-database)) - **Unpublish** - a warning is shown for any default Semantic Models created by the Mirror Database. This is a current limitation of the Fabric API and can be ignored. ## ML Experiment - **Parameterization:** - The `find_replace` section in the `parameter.yml` file is not applied. - **Only the ML Shell is created.** The create API does not support the creation an machine learning experiment with definition. ## Mounted Data Factory - **Parameterization:** - The `find_replace` section in the `parameter.yml` file is not applied. - Deploys a **mounted** Azure Data Factory to a Fabric Workspace. - Before deployment, ensure the Azure Data Factory resource exists with proper permissions. ## Notebook - **Parameterization:** - Notebooks attached to lakehouses always point to the original lakehouse unless parameterized in the `find_replace` section of the `parameter.yml` file. - **Resources** are not source controlled and will not be deployed. - **Note:** Both `.py` and `.ipynb` formats are supported. Git-integrated Notebooks only support `.py`, while Notebooks exported via the Get Item Definition API support both formats. ## Ontology - **Parameterization:** - Referenced items that exist in a different workspace will always point to the original item unless parameterized in the `find_replace` section of the `parameter.yml` file. - Referenced items within the same workspace are automatically re-pointed to the new item in the target workspace. ## Real-Time Dashboard - **Parameterization:** - Real-Time Dashboard attached to KQL databases always point to the original KQL database unless parameterized in the `find_replace` section of the `parameter.yml` file. - The **cluster/query URI** of the KQL database(s) used in the dashboard must be present in the Real-Time Dashboard JSON for rebinding. If the Real-Time Dashboard is attached to a KQL database within the same workspace, the cluster URI value is empty and needs to be re-added. `fabric ci-cd` handles this automatically. ## Report - **Parameterization:** - Reports connected to Semantic Models outside of the same workspace always point to the original Semantic Model unless parameterized in the `parameter.yml` file. - Reports with `byPath` references to Semantic Models within the same workspace are automatically re-pointed to the new item in the target workspace. - Reports with `byConnection` references to Semantic Models within or outside of the same workspace can be parameterized using `find_replace` or `key_value_replace` parameters to update workspace IDs, semantic model names, and semantic model IDs in the connection string. See [Reports Parameterization Example](parameterization.md#reports). ## Semantic Model - **Parameterization:** - Semantic Models connected to sources outside of the same workspace always point to the original item unless parameterized in the `find_replace` section of the `parameter.yml` file. - Semantic Models connected to sources within the same workspace may or may not be re-pointed; it is best to test this before taking a dependency. Use the `find_replace` section of the `parameter.yml` file as needed. - **Automatic Connection Binding:** - Use the `semantic_model_binding` parameter in `parameter.yml` to automatically bind Semantic Model connections after deployment, removing the need for manual connection configuration on initial deployment. - **Note:** Only a single connection binding per Semantic Model is currently supported. If your Semantic Model uses multiple connections, additional connections must be configured manually after deployment. - See [Parameterization -> semantic_model_binding](parameterization.md#semantic_model_binding) for configuration details. - **Initial deployment** requires manual configuration of the connection after deployment **unless** `semantic_model_binding` is configured in the `parameter.yml` file. ## Spark Job Definition - **Parameterization:** - Spark Job Definitions attached to lakehouses always point to the original lakehouse unless parameterized in the `find_replace` section of the `parameter.yml` file. - When connected to an Environment within the same workspace, the deployed Spark Job Definition will be re-pointed to the new Environment in the target workspace. - **File Types Supported:** The v2 API which is used only supports Spark Job Definitions with file formats of `.py` or `.scala`. The `.jar` file format isn't supported. ## SQL Database - **Parameterization:** - The `find_replace` section in the `parameter.yml` file is not applied. - **SQL Database content is not deployed.** Only the item shell is deployed. The SQL database schema (DDL) must be deployed separately using a DACPAC or other tools such as dbt. - **Unpublish** is disabled by default, enable with feature flag `enable_sqldatabase_unpublish`. ## User Data Function - **Parameterization:** - Connection references and libraries in the `definitions.json` file will always point to the original items unless parameterized in the `find_replace` section of the `parameter.yml` file. - **Important:** When parameterizing connections or libraries that reference items in different workspaces, use the appropriate `replace_value` variables with workspace and item IDs. - **Deployment**: - Includes all connections and libraries specified in the source workspace. - **Important:** Due to the nature of UserDataFunctions they could take up to a few minutes to publish. - **Metadata and configuration** are automatically managed through the `functions.json` file in the resources folder. ## Variable Library - **Parameterization:** - The active value set of the variable library is defined by the `environment` field passed into the `FabricWorkspace` object. If no `environment` is specified, the active Value Set will not be changed. - **Changing Value Sets:** - Variable Libraries do not support programmatically changing the name of value set which is active - After the initial deployment, if an active set is renamed, or removed, the deployment will fail - Manual intervention will be required to make the necessary changes in the Fabric UI and then restart the deployment ## Warehouse - **Parameterization:** - The `find_replace` section in the `parameter.yml` file is not applied. - **Warehouse content is not deployed.** Only the item shell is deployed. Warehouse DDL must be deployed separately using a DACPAC or other tools such as dbt. - **Case insensitive collation is supported** custom collation must be manually edited in the `.platform` file creation payload. See [How to: Create a warehouse with case-insensitive (CI) collation](https://learn.microsoft.com/en-us/fabric/data-warehouse/collation) for more details. - **Unpublish** is disabled by default, enable with feature flag `enable_warehouse_unpublish`. ================================================ FILE: docs/how_to/optional_feature.md ================================================ # Optional Features fabric-cicd has an expected default flow; however, there will always be cases where overriding default behavior is required. ## Feature Flags For scenarios that aren't supported by default, fabric-cicd offers feature flags. Below is an exhaustive list of currently supported features. | Flag Name | Description | Experimental | | ----------------------------------------- | --------------------------------------------------------------------------------------------------------- | ------------ | | `enable_lakehouse_unpublish` | Set to enable the deletion of Lakehouses | | | `enable_warehouse_unpublish` | Set to enable the deletion of Warehouses | | | `enable_sqldatabase_unpublish` | Set to enable the deletion of SQL Databases | | | `enable_eventhouse_unpublish` | Set to enable the deletion of Eventhouses | | | `enable_kqldatabase_unpublish` | Set to enable the deletion of KQL Databases (attached to Eventhouses) | | | `enable_shortcut_publish` | Set to enable deploying shortcuts with the Lakehouse | | | `enable_environment_variable_replacement` | Set to enable the use of pipeline variables | | | `disable_workspace_folder_publish` | Set to disable deploying workspace sub folders | | | `enable_experimental_features` | Set to enable experimental features, such as selective deployments | | | `enable_items_to_include` | Set to enable selective publishing/unpublishing of items | ☑️ | | `enable_exclude_folder` | Set to enable folder-based exclusion during publish operations | ☑️ | | `enable_include_folder` | Set to enable folder-based inclusion during publish operations | ☑️ | | `enable_shortcut_exclude` | Set to enable selective publishing of shortcuts in a Lakehouse | ☑️ | | `enable_response_collection` | Set to enable collection of API responses during publish and unpublish operations | | | `continue_on_shortcut_failure` | Set to allow deployment to continue even when shortcuts fail to publish | | | `enable_hard_delete` | Set to enable hard deletion of items, bypassing the workspace recycle bin. Requires workspace Admin role. | | Example ```python from fabric_cicd import append_feature_flag append_feature_flag("enable_lakehouse_unpublish") append_feature_flag("enable_warehouse_unpublish") append_feature_flag("enable_environment_variable_replacement") append_feature_flag("enable_response_collection") ``` ## Selective Deployment Features By default, fabric-cicd performs a full deployment of all repository items. Selective deployment is an experimental feature due to the risk of deploying Fabric items that have dependencies on other items, which can result in broken deployments. These features support a range of filtering options, from broader folder-based selection to more granular item-level and shortcut-level filtering. To use these features, you must enable both the `enable_experimental_features` flag and the specific feature flag (if applicable). **Warning:** Selective deployment is not recommended due to potential issues with dependency management. ### Folder-Level Filtering A subset of items in the repository that exist within a Fabric workspace folder can be published using one of the following experimental features. Only one of these features can be applied during a deployment. Use case: selectively deploy a **group** of Fabric items (must be contained within folders). Folder-based item exclusion/inclusion is not supported in the unpublish scenario. 1. **`folder_path_exclude_regex`** - Optional parameter in `publish_all_items()`, set to a regex pattern that matches Fabric folder path(s) containing items in the repository. - Requires the `enable_exclude_folder` feature flag. - The folder path(s) and items contained within that match the regex will be excluded from the publish operation. - When using `folder_path_exclude_regex`, the pattern is matched using `search()` (substring match), so a pattern like `subfolder1` will match any folder path containing "subfolder1" (e.g., `/subfolder1`, `/subfolder1/subfolder2`, `/other/subfolder1`). - To target a specific folder, use an anchored pattern (e.g., `^/subfolder1$`) — this ensures only the exact folder path matches. - Child folders like `/subfolder1/subfolder2` will also be excluded automatically since their parent folder was excluded, preserving a consistent folder hierarchy. 2. **`folder_path_to_include`** - Optional parameter in `publish_all_items()`, set to a list of strings that exactly match the folder path(s) containing items in the repository. - Requires the `enable_include_folder` feature flag. - Folder paths must start with `/` (e.g., `/folder_name` or `/folder_name/nested_folder`). The matching folder path(s) and their contained items will be included in the publish operation; any other items contained within Fabric folders will be excluded. - When using `folder_path_to_include` with nested paths (e.g., `/subfolder1/subfolder2`), ancestor folders (e.g., `/subfolder1`) are automatically created to preserve the correct folder hierarchy, but items directly under the ancestor folder are **not** published unless the ancestor folder is also explicitly included in the list. **Note:** `folder_path_exclude_regex` and `folder_path_to_include` are mutually exclusive and cannot be used together for the same deployment. These filters are ignored when the `disable_workspace_folder_publish` feature flag is set. Folder-based filtering does not impact standalone items. ### Item-Level Filtering A subset of items in the repository can be published/unpublished using one of the following features. Both features are technically supported, but **it is recommended to use one feature per deployment to avoid unexpected results**. 1. **`item_name_exclude_regex`** - Optional parameter in `publish_all_items()` and `unpublish_all_orphan_items()`, set to a regex pattern that matches item name(s) found in the repository. - **This feature does not require any feature flags.** - Fabric items that match the regex will be excluded from the publish/unpublish operation. 2. **`items_to_include`** - Optional parameter in `publish_all_items()` and `unpublish_all_orphan_items()`, set to a list of strings that exactly match items in the repository. - Requires the `enable_items_to_include` feature flag. - Must be in the format: `"item_name.item_type"`. The matching item(s) will be included in the publish/unpublish operation. **Note:** `item_name_exclude_regex` and `items_to_include` can be applied to items within Fabric folders or standalone items. Item-level filtering can be combined with folder-level filtering, but be cautious when using both to avoid unexpected results. ### Filter Precedence Filters are evaluated in the following order: 1. **`items_to_include`** — Scope is narrowed upfront; only explicitly listed items proceed to further checks 2. **`item_name_exclude_regex`** — Items matching the regex are excluded 3. **`folder_path_exclude_regex`** — Items in matching folders are excluded 4. **`folder_path_to_include`** — Only items in specified folders are published **Note:** `folder_path_exclude_regex` and `folder_path_to_include` are mutually exclusive — only one can be used per deployment. Standalone items (items not in any folder) are not impacted by folder-level filters. When `items_to_include` is combined with exclusion filters, an item must first be in the include list before exclusion filters are evaluated against it. ### Lakehouse Shortcut Filtering Shortcuts are items associated with Lakehouse items and can be selectively published using the following experimental feature: 1. **`shortcut_exclude_regex`** - Optional parameter in `publish_all_items()`, set to a regex pattern that matches the shortcut name(s) found within Lakehouse item(s) in the repository. - Requires the `enable_shortcut_exclude` feature flag. - The matching shortcut(s) will be excluded from publishing. **Note:** This feature can be applied along with the other selective deployment features — please be cautious when using to avoid unexpected results. ## Git-Based Change Detection `get_changed_items()` is a public utility function that uses `git diff` to detect which Fabric items have been added, modified, or renamed relative to a given git reference. It returns a list of strings in `"item_name.item_type"` format that can be passed directly to `items_to_include` in `publish_all_items()`. While `get_changed_items()` itself requires no feature flags, passing its output to `items_to_include` requires the experimental feature flags. **Important:** If `get_changed_items()` returns an empty list (no changes detected), do not call `publish_all_items()` without an explicit `items_to_include` list, as this would default to a full deployment. Always guard against the empty-list case: ```python from fabric_cicd import FabricWorkspace, publish_all_items, get_changed_items, append_feature_flag append_feature_flag("enable_experimental_features") append_feature_flag("enable_items_to_include") workspace = FabricWorkspace( workspace_id="your-workspace-id", repository_directory="/path/to/repo", item_type_in_scope=["Notebook", "DataPipeline"], token_credential=token_credential, # or any other TokenCredential ) changed = get_changed_items(workspace.repository_directory) if changed: publish_all_items(workspace, items_to_include=changed) else: print("No changed items detected — skipping deployment.") ``` To compare against a branch or a specific commit instead of the previous commit, pass a custom `git_compare_ref`: ```python changed = get_changed_items(workspace.repository_directory, git_compare_ref="main") ``` **Note:** `get_changed_items()` returns only items that were **modified or added** (i.e., candidates for publishing). It does not return deleted items. Passing `items_to_include` to `publish_all_items()` requires enabling the `enable_experimental_features` and `enable_items_to_include` feature flags. ## Debugging If an error arises, or you want full transparency to all calls being made outside the library, enable debugging. Enabling debugging will write all API calls to the terminal. The logs can also be found in the `fabric_cicd.error.log` file. ```python from fabric_cicd import change_log_level change_log_level("DEBUG") ``` **Note:** The `"DEBUG"` parameter can be omitted as it is the default value. For comprehensive debugging information, including how to use the error log file and debug scripts, see the [Troubleshooting Guide](troubleshooting.md). ================================================ FILE: docs/how_to/parameterization.md ================================================ # Parameterization ## Overview To handle environment-specific values committed to git, use a `parameter.yml` file. This file supports programmatically changing values based on the `environment` field passed into the `FabricWorkspace` object. If the environment value is not found in the `parameter.yml` file, any dependent replacements will be skipped. This file should sit in the root of the `repository_directory` folder specified in the FabricWorkspace object. Example of parameter.yml location based on provided repository directory: ```python from azure.identity import AzureCliCredential from fabric_cicd import FabricWorkspace token_credential = AzureCliCredential() workspace = FabricWorkspace( workspace_id="your-workspace-id", repository_directory="C:/dev/workspace", environment="PROD", item_type_in_scope=["Notebook"], token_credential=token_credential, # or any other TokenCredential ) ``` ``` C:/dev/workspace /HelloWorld.Notebook ... /GoodbyeWorld.Notebook ... /parameter.yml ``` Example of parameter.yml file content: ```yaml find_replace: - find_value: "your-dev-lakehouse-id" replace_value: PPE: "ppe-lakehouse-id" PROD: "prod-lakehouse-id" key_value_replace: - find_key: $.variables[?(@.name=="Environment")].value replace_value: PPE: "PPE" PROD: "PROD" spark_pool: - instance_pool_id: "your-dev-pool-instance-id" replace_value: PPE: type: "Capacity" name: "PPE-Pool-name" PROD: type: "Capacity" name: "PROD-Pool-name" semantic_model_binding: default: connection_id: PPE: "PPE-connection_id" PROD: "PROD-connection_id" models: - semantic_model_name: "semantic_model_name" connection_id: PPE: "PPE-connection_id" PROD: "PROD-connection_id" ``` Raise a [feature request](https://github.com/microsoft/fabric-cicd/issues/new?template=2-feature.yml) for additional parameterization capabilities. ## Parameter Inputs ### `find_replace` For generic find-and-replace operations. This will replace every instance of a specified string in every file. Specify the `find_value` and the `replace_value` for each environment (e.g., PPE, PROD). Optional fields, including `item_type`, `item_name`, and `file_path`, can be used as file filters for more fine-grained control over where the replacement occurs. The `is_regex` field can be added and set to `"true"` to enable regex pattern matching for the `find_value`. Note: A common use case for this function is to replace values in text based file types like notebooks. ```yaml find_replace: # Required fields: value must be a string - find_value: replace_value: : : # Optional fields # Set to "true" to treat find_value as a regex pattern is_regex: "" # Filter values must be a string or array of strings item_type: item_name: file_path: ``` ### `key_value_replace` Provides the ability to perform key based replacement operations in JSON and YAML files. This will look for a specific key using a valid JSONPath expression and replace every found instance in every file. Specify the `find_key` and the `replace_value` for each environment (e.g., PPE, PROD). Optional fields, including `item_type`, `item_name`, and `file_path`, can be used as file filters for more fine-grained control over where the replacement occurs. Refer to https://jsonpath.com/ for a simple to use JSONPath evaluator. Note: A common use case for this parameter is to replace values in key/value file types like Data Pipeline, Schedule files, etc. The `key_value_replace` parameter automatically detects and processes any file containing valid JSON or YAML content, regardless of file extension (e.g., `.schedules` files). This does not apply to `.platform` files, which are intentionally excluded from parameterization to maintain item integrity. The `replace_value` field supports the same dynamic replacement variables as `find_replace`, including `$items` and `$workspace` notation. See the **Dynamic Replacement** section under `find_replace` for details on supported variables and attributes. ```yaml key_value_replace: # Required fields: key must be JSONPath - find_key: replace_value: : : # Optional fields: value must be a string or array of strings item_type: item_name: file_path: ``` Example with `$items` notation: ```yaml key_value_replace: - find_key: $.properties.activities[?(@.name=="Run Notebook")].typeProperties.notebookId replace_value: PPE: "$items.Notebook.Hello World.$id" # PPE Hello World Notebook GUID PROD: "$items.Notebook.Hello World.$id" # PROD Hello World Notebook GUID item_type: "DataPipeline" - find_key: $.properties.activities[?(@.name=="Run Notebook")].typeProperties.workspaceId replace_value: PPE: "$workspace.$id" # PPE workspace ID PROD: "$workspace.$id" # PROD workspace ID item_type: "DataPipeline" ``` ### `spark_pool` Environments attached to custom spark pools need to be parameterized because the `instance_pool_id` in the `Sparkcompute.yml` file is workspace-specific and must be mapped to the target workspace's pool. Provide the `instance_pool_id` value, and the pool `type` and `name` values as the `replace_value` for each environment (e.g., PPE, PROD). An optional field, `item_name`, can be used to filter the specific environment item where the replacement will occur. ```yaml spark_pool: # Required fields: value must be a string - instance_pool_id: replace_value: : type: name: : type: name: # Optional field: value must be a string or array of strings item_name: ``` ### `semantic_model_binding` Semantic model binding automatically connects semantic models to the appropriate data source connection (e.g., cloud or gateway/on-premises) after deployment, ensuring your models can refresh data in the target environment. ```yaml semantic_model_binding: # Default connection for all models not explicitly listed default: connection_id: PPE: PROD: # OR use _ALL_ for same connection across environments # _ALL_: # Explicit bindings override default models: - semantic_model_name: "" connection_id: PPE: PROD: - semantic_model_name: ["", "", ...] connection_id: _ALL_: ``` **Notes:** - The `_ALL_` environment key (case-insensitive) can be used in the `connection_id` dictionary to apply the same connection to any target environment. - Connection ID values must be valid GUIDs. - **Only a single connection binding per Semantic Model is currently supported.** If your Semantic Model uses multiple connections (e.g., connecting to both a SQL database and a Lakehouse), only one can be configured through `semantic_model_binding`. Additional connections must be configured manually after deployment. ## Advanced Find and Replace ### `find_value` Regex In the `find_replace` parameter, the `find_value` can be set to a regex pattern instead of a literal string to find a value in the files to replace. When a match is found, the `find_value` is assigned to the matched string and can be used to replace all occurrences of that value in the file. - **How to** use this feature: - Set the `find_value` to a **valid regex pattern** wrapped in quotes. - Include the optional field `is_regex` and set it to the value `"true"`, see [more details](#regex-pattern-match). - **Important:** - The user is solely **responsible for providing a valid and correctly matching regex pattern**. If the pattern is invalid (i.e., it cannot be compiled) or fails to match any content in the target files, deployment will fail. - A valid regex pattern requires the following: - Ensure that all special characters in the regex pattern are properly **escaped**. - The exact value intended to be replaced must be enclosed in parentheses `( )`. - The parentheses creates a **capture group 1**, which must always be used as the replacement target. Capture group 1 should isolate values like a GUID, SQL connection string, etc. - Include the **surrounding context** in the pattern, such as property/field names, quotes, etc. to ensure it matches the correct value and not a value with a similar format elsewhere in the file. - **Example:** - Use a regex `find_value` to match a lakehouse ID inside a Notebook file. **Note:** avoid using a pattern that ONLY matches the GUID format as doing so would risk replacing any matching GUID in the file, not just the intended one. Include the surrounding context in your pattern—such as `# META "default_lakehouse": "123456"`—and capture only the `123456` GUID in group 1. This ensures that only the correct, context-specific GUID is replaced. ```yaml find_replace: # A valid regex pattern to match the default_lakehouse ID - find_value: \#\s*META\s+"default_lakehouse":\s*"([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})" replace_value: PPE: "e8a7f3c6-9b2d-4f5e-a1b0-7c98d4e6a5f3" # PPE Lakehouse GUID PROD: "12c45d67-89ab-4cde-f012-3456789abcde" # PROD Lakehouse GUID # Optional field: Set to "true" to treat find_value as a regex pattern is_regex: "true" # "" item_type: "Notebook" # filter on notebook files item_name: ["Hello World", "Goodbye World"] # filter on specific notebook files ``` ### Dynamic Replacement The `replace_value` field in the `find_replace` and `key_value_replace` parameters supports fabric-cicd defined _variables_ that reference workspace or deployed item metadata: - **Dynamic workspace/item metadata replacement ONLY works for referenced items that exist in the `repository_directory`.** - Dynamic replacement works in tandem with `find_value` (for `find_replace`) as either a regex or a literal string, or with `find_key` (for `key_value_replace`) as a JSONPath expression. - The `replace_value` can contain a mix of input values within the _same_ parameter input, e.g. `PPE` is set to a static string and `PROD` is set to a variable. - **Supported variables:** - **Workspace variable:** | Workspace Variable | Description | Example | | --------------------------------------------------------------- | ----------------------------------------------------------------------------------------- | ---------------------------------------------------------- | | `$workspace.$id` or `$workspace.id` | Workspace ID of the target environment | `$workspace.$id` or `$workspace.id` | | `$workspace.$name` | Display name of the target workspace | `$workspace.$name` | | `$workspace.$name_encoded` | URL-encoded display name of the target workspace (spaces become `%20`, etc.) | `$workspace.$name_encoded` | | `$workspace..$id` or `$workspace.` | Workspace ID of the specified workspace name | `$workspace.TestWorkspace.$id` or `$workspace.TestWorkspace` | | `$workspace..$items...$` | Attribute value of the specified item in a specified workspace (see supported attributes) | `$workspace.TestWorkspace.$items.Lakehouse.Example_LH.$id` | > **Notes:** > > - When using `$workspace.$name`, `$workspace.$name_encoded`, `$workspace..$id` / `$workspace.`, or `$workspace..$items...$`, ensure the executing identity has proper permissions to access the relevant workspace. > - For `$workspace..$id` / `$workspace.` and `$workspace..$items...$`, the provided workspace display name must be an exact, case-sensitive match. > - `$workspace.$name` returns the raw workspace display name. If the target value must remain URL-encoded, use `$workspace.$name_encoded` instead, which automatically percent-encodes the name (e.g., `My Workspace` becomes `My%20Workspace`). - **Item attribute variable:** replaces the item's attribute value with the corresponding attribute value of the item in the deployed/target workspace. - `$items...` (legacy format) - **`$items...$`** (new format) - **Supported attributes:** | Attribute Variable | Supported Items | Example | Sample Replace Value | | ------------------------------------------------- | --------------------------------- | ------------------------------------------------- | -------------------------------------------------------------- | | `$items...$id` | All | `$items.Notebook.MyNotebook.$id` | `123e4567-e89b-12d3-a456-426614174000` | | `$items...$sqlendpoint` | Lakehouse, SQLDatabase, Warehouse | `$items.Lakehouse.MyLakehouse.$sqlendpoint` | `abc123def456.datawarehouse.fabric.microsoft.com` | | `$items...$sqlendpointid` | Lakehouse | `$items.Lakehouse.MyLakehouse.$sqlendpointid` | `37dc8a41-dea9-465d-b528-3e95043b2356` | | `$items...$queryserviceuri` | Eventhouse | `$items.Eventhouse.MyEventhouse.$queryserviceuri` | `https://trd-a1b2c3d4e5f6g7h8i9.z4.kusto.fabric.microsoft.com` | - Attributes should be **lowercase**. - Item type and name are **case-sensitive**. - Item type must be valid and in scope. - Item name must be an **exact match** (include spaces, if present). - **Example:** set `$items.Notebook.Hello World.$id` to get the item ID of the `"Hello World"` Notebook in the target workspace. - **Important**: Deployment will fail in the following cases: - Incorrect variable syntax used, e.g., `$item.Notebook.Hello World.$id` instead of `$items.Notebook.Hello World.$id`. - The specified **item type** or **name** is invalid or does NOT exist in the deployed workspace, e.g., `$items.Notebook.HelloWorld.$id` or `$items.Environment.Hello World.$id`. - An invalid attribute name is provided, e.g., `$items.Notebook.Hello World.$guid` instead of `$items.Notebook.Hello World.$id`. - The attribute value does NOT exist, e.g., `$items.Notebook.Hello World.$sqlendpoint` (Notebook items don't have a SQL Endpoint). - For example use-cases, see the **Notebook/Dataflow Advanced `find_replace` Parameterization Case.** ### Environment Variable Replacement In the `find_replace` parameter, if the `enable_environment_variable_replacement` feature flag is set, pipeline/environment variables will be used to replace the values in the `parameter.yml` file with the corresponding values from the variables dictionary. **Only Environment Variable beginning with '$ENV:' will be used as replacement values.** See example below: ```yaml find_replace: # Lakehouse GUID - find_value: "db52be81-c2b2-4261-84fa-840c67f4bbd0" replace_value: PPE: "$ENV:ppe_lakehouse" PROD: "$ENV:prod_lakehouse" ``` ### File Filters File filtering is supported in all parameters. This feature is optional and can be used to specify the files where replacement is intended to occur. - **Supported filters:** `item_type`, `item_name`, and `file_path`, see [more details](#supported-file-filters). - **Note:** only `item_name` filter is supported in `spark_pool` parameter. - **Expected behavior:** - If at least one filter value does not match, the replacement will be skipped for that file. - If none of the optional filter fields or values are provided, the value found in _any_ repository file is subject to replacement. - **Filter input:** - Input values are **case sensitive**. - Input values must be **string** or **array** (enables one or many values to filter on). - YAML supports array inputs using bracket ( **[ ]** ) or dash ( **-** ) notation. find_replace/key_value_replace ```yaml : # Required fields: value must be a string - : replace_value: : : # Optional fields # Filter values must be a string or array of strings item_type: item_name: file_path: ``` spark_pool ```yaml spark_pool: # Required fields: value must be a string - instance_pool_id: replace_value: : type: name: : type: name: # Optional field: value must be a string or array of strings item_name: ``` ### \_ALL\_ Environment Key in `replace_value` The `_ALL_` environment key (case-insensitive) in `replace_value` is supported for all parameter types (`find_replace`, `key_value_replace`, `spark_pool`) and applies the replacement to any target environment. When `_ALL_` is used, it must be the only environment key in the `replace_value` dictionary. Using `ALL` without underscores will be treated as a regular environment key. Use case: when the same replacement value applies to all target environments (particularly valuable in dynamic replacement scenarios). ```yaml find_replace: # Lakehouse GUID - find_value: "db52be81-c2b2-4261-84fa-840c67f4bbd0" replace_value: # use _ALL_ or _all_ or _All_ _ALL_: "$items.Lakehouse.Example_LH.$id" ``` ## Optional Fields When optional fields are omitted or left empty, only basic parameterization functionality will be available. To enable advanced features, you must add the specific optional field(s) (if applicable) and set appropriately. **Important:** - String input values should be wrapped in quotes. Remember to escape special characters, such as **\\** in `file_path` inputs. - `is_regex` and filter fields can be used in the same parameter configuration. ### Regex Pattern Match #### `is_regex` - Only applicable to the `find_replace` parameter. - Include `is_regex` field when setting the `find_value` to a **valid regex pattern.** - When the `is_regex` field is set to the **string** value `"true"` or `"True"` (case-insensitive), regex pattern matching is enabled. - When regex pattern matching is enabled, the `find_value` is interpreted as a regex pattern rather than a literal string. ### Supported File Filters #### `item_type` - Item types must be valid and within scope of deployment. - See valid [types](https://learn.microsoft.com/en-us/rest/api/fabric/core/items/create-item?tabs=HTTP#itemtype). #### `item_name` - Item names must match the exact names of items in the `repository_directory`. #### `file_path` - `file_path` accepts three types of paths within the _repository directory_ boundary: - **Absolute paths:** Full path starting from the drive root. - **Relative paths:** Paths relative to the _repository directory_. - **Wildcard paths:** Paths containing glob patterns. - When using _wildcard paths_: - Common patterns include `*` (matches any characters in a filename), `**` (matches any directory depth). - All matched files must exist within the _repository directory_. - When using wildcard patterns, verify your syntax carefully to avoid unexpected matching behavior. - **Examples:** `**/notebook-content.py` matches all notebook files in the repository directory, `Sample Pipelines/*.json` matches json files in the Sample Pipelines folder in the repository directory. ## Parameter File Validation Validation of the `parameter.yml` file is a built-in feature of fabric-cicd, managed by the `Parameter` class. Validation is utilized in the following scenarios: **Debuggability:** Users can debug and validate their parameter file to ensure it meets the acceptable structure and input value criteria before running a deployment. Simply run the `debug_parameterization.py` script located in the `devtools` directory. **Deployment:** At the start of a deployment, an automated validation checks the validity of the `parameter.yml` file, if it is present. This step ensures that valid parameters are loaded, allowing deployment to run smoothly with correctly applied parameterized configurations. If the parameter file is invalid, the deployment will NOT proceed. ## Parameter File Templates This option supports splitting a large parameter file into smaller parameter file "templates". Create template YAML files in any location (in the following example, the files are located in a `templates` directory within the repository directory). In the main `parameter.yml` file, add the `extend` key with a list of template parameter file paths **relative to the main parameter file location.** Repository directory ``` C:/dev/workspace /HelloWorld.Notebook ... /GoodbyeWorld.Notebook ... /parameter.yml ... /templates /nb_parameters.yml /pl_parameters.yml /df_parameters.yml ``` Main `parameter.yml` file ```yaml extend: - "./templates/nb_parameters.yml" # - "./templates/pl_parameters.yml" # - "./templates/df_parameters.yml" find_replace: # Lakehouse Connection Guid - find_value: "db52be81-c2b2-4261-84fa-840c67f4bbd0" replace_value: PPE: "81bbb339-8d0b-46e8-bfa6-289a159c0733" PROD: "5d6a1b16-447f-464a-b959-45d0fed35ca0" # Optional fields: item_type: "Notebook" item_name: ["Hello World", "Hello World Subfolder"] file_path: - "/Hello World.Notebook/notebook-content.py" - "/subfolder/Hello World Subfolder.Notebook/notebook-content.py" spark_pool: # CapacityPool_Large - instance_pool_id: "72c68dbc-0775-4d59-909d-a47896f4573b" replace_value: PPE: type: "Capacity" name: "CapacityPool_Large_PPE" PROD: type: "Capacity" name: "CapacityPool_Large_PROD" # Optional field: item_name: "World" ``` `nb_parameters.yml` file ```yaml find_replace: # Lakehouse Connection Guid regex - find_value: \#\s*META\s+"default_lakehouse":\s*"([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})" replace_value: # Variable: $items.type.name.attribute (Note: item type and name values are CASE SENSITIVE; id attribute returns the deployed item's id/guid) PPE: "$items.Lakehouse.WithoutSchema.id" PROD: "$items.Lakehouse.WithoutSchema.id" # Optional fields: is_regex: "true" file_path: "/Example Notebook.Notebook/notebook-content.py" # Lakehouse workspace id regex - find_value: \#\s*META\s+"default_lakehouse_workspace_id":\s*"([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})" replace_value: # Variable: $workspace.id -> target workspace id PPE: "$workspace.id" PROD: "$workspace.id" # Optional fields: is_regex: "true" file_path: "/Example Notebook.Notebook/notebook-content.py" ``` Parameter dictionary ```json { "find_replace": [ { "find_value": "db52be81-c2b2-4261-84fa-840c67f4bbd0", "replace_value": { "PPE": "81bbb339-8d0b-46e8-bfa6-289a159c0733", "PROD": "5d6a1b16-447f-464a-b959-45d0fed35ca0" }, "item_type": "Notebook", "item_name": ["Hello World", "Hello World Subfolder"], "file_path": [ "/Hello World.Notebook/notebook-content.py", "/subfolder/Hello World Subfolder.Notebook/notebook-content.py" ] }, { "find_value": "\\#\\s*META\\s+\"default_lakehouse\":\\s*\"([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})\"", "replace_value": { "PPE": "$items.Lakehouse.WithoutSchema.id", "PROD": "$items.Lakehouse.WithoutSchema.id" }, "is_regex": "true", "file_path": "/Example Notebook.Notebook/notebook-content.py" }, { "find_value": "\\#\\s*META\\s+\"default_lakehouse_workspace_id\":\\s*\"([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})\"", "replace_value": { "PPE": "$workspace.id", "PROD": "$workspace.id" }, "is_regex": "true", "file_path": "/Example Notebook.Notebook/notebook-content.py" } ], "spark_pool": [ { "instance_pool_id": "72c68dbc-0775-4d59-909d-a47896f4573b", "replace_value": { "PPE": { "type": "Capacity", "name": "CapacityPool_Large_PPE" }, "PROD": { "type": "Capacity", "name": "CapacityPool_Large_PROD" } }, "item_name": "World" } ] } ``` ## Sample Parameter File An exhaustive example of all capabilities currently supported in the `parameter.yml` file. ```yaml find_replace: - find_value: "123e4567-e89b-12d3-a456-426614174000" # lakehouse GUID to be replaced replace_value: PPE: "f47ac10b-58cc-4372-a567-0e02b2c3d479" # PPE lakehouse GUID PROD: "9b2e5f4c-8d3a-4f1b-9c3e-2d5b6e4a7f8c" # PROD lakehouse GUID item_type: "Notebook" # filter on notebook files item_name: ["Hello World", "Goodbye World"] # filter on specific notebook files # enable_environment_variable_replacement feature flag to replace workspace ID - find_value: "8f5c0cec-a8ea-48cd-9da4-871dc2642f4c" # workspace ID to be replaced replace_value: PPE: "$ENV:ppe_workspace_id" # PPE workspace ID (ENV variable) PROD: "$ENV:prod_workspace_id" # PROD workspace ID (ENV variable) file_path: # filter on notebook files with these paths - "/Hello World.Notebook/notebook-content.py" - "\\Goodbye World.Notebook\\notebook-content.py" # lakehouse GUID to be replaced (using regex pattern) - find_value: \#\s*META\s+"default_lakehouse":\s*"([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})" replace_value: PPE: "$items.Lakehouse.Example_LH.$id" # PPE lakehouse GUID (dynamic) PROD: "$items.Lakehouse.Example_LH.$id" # PROD lakehouse GUID (dynamic) is_regex: "true" # enable regex pattern matching item_type: "Notebook" # filter on notebook files item_name: ["Hello World", "Goodbye World"] # filter on specific notebook files # lakehouse workspace ID to be replaced (using regex pattern) - find_value: \#\s*META\s+"default_lakehouse_workspace_id":\s*"([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})" replace_value: _ALL_: "$workspace.$id" # workspace ID of the target environment (dynamic) is_regex: "true" # enable regex pattern matching item_name: # filter on specific notebook files - "Hello World" - "Goodbye World" file_path: "**/notebook-content.py" # filter on notebook files using wildcard paths key_value_replace: # SQL Server Connection to be replaced - find_key: $.properties.activities[?(@.name=="Load_Intake")].typeProperties.source.datasetSettings.externalReferences.connection replace_value: PPE: "6c517159-d27a-41d5-b71e-ca1ecff6542b" # PPE SQL Server Connection PROD: "6c517159-d27a-41d5-b71e-ca1ecff6542b" # PROD SQL Server Connection item_type: "DataPipeline" # filter on data pipeline files # Schedule enabled state to be replaced - find_key: $.schedules[?(@.jobType=="Execute")].enabled replace_value: PPE: false # disable execution in PPE environment PROD: true # enable execution in PROD environment file_path: "**/.schedules" # filter on all .schedules files spark_pool: - instance_pool_id: "72c68dbc-0775-4d59-909d-a47896f4573b" # spark_pool_instance_id to be replaced replace_value: PPE: type: "Capacity" # target spark pool type, only supports Capacity or Workspace name: "CapacityPool_Medium" # target spark pool name PROD: type: "Capacity" # target spark pool type, only supports Capacity or Workspace name: "CapacityPool_Large" # target spark pool name item_name: "World" # filter on environment file for environment named "World" - instance_pool_id: "e7b8f1c4-4a6e-4b8b-9b2e-8f1e5d6a9c3d" # spark_pool_instance_id to be replaced replace_value: PPE: type: "Workspace" # target spark pool type, only supports Capacity or Workspace name: "WorkspacePool_Medium" # target spark pool name item_name: ["World_1", "World_2", "World_3"] # filter on environment files for environments with these names ``` ## Examples by Item Type ### Notebooks #### `find_replace` Parameterization Case **Case:** A Notebook is attached to a Lakehouse which resides in different workspaces. The Workspace and Lakehouse GUIDs in the Notebook need to be updated to ensure the Notebook points to the correct Lakehouse once deployed. **Solution:** In the `notebook-content.py` file, the default_lakehouse `47592d55-9a83-41a8-9b21-e1ef44264161`, and default_lakehouse_workspace_id `2190baad-a374-4114-addd-0dcf0533e69d` must be replaced with the corresponding GUIDs of the Lakehouse in the target environment (PPE/PROD/etc). This replacement is managed by the `find_replace` input in the `parameter.yml` file where fabric-cicd finds every instance of the string within the specified repository files and replaces it with the string for the deployed environment. parameter.yml file ```yaml find_replace: - find_value: "47592d55-9a83-41a8-9b21-e1ef44264161" # lakehouse GUID to be replaced replace_value: PPE: "a21e502a-51a5-4455-bb3d-6faf1e3e21fb" # PPE lakehouse GUID PROD: "1069f2ff-bb30-42a0-97b3-1f4655705b8a" # PROD lakehouse GUID item_type: "Notebook" # filter on notebook files item_name: ["Hello World", "Goodbye World"] # filter on specific notebook files - find_value: "2190baad-a374-4114-addd-0dcf0533e69d" # workspace ID to be replaced replace_value: PPE: "5a6ebbe6-9289-4105-b47c-cf158247b911" # PPE workspace ID PROD: "f9e8cbe0-2669-4e06-a026-7c75e5af8107" # PROD workspace ID file_path: # filter on notebook files with these paths - "/Hello World.Notebook/notebook-content.py" - "\\Goodbye World.Notebook\\notebook-content.py" ``` notebook-content.py file ```python # Fabric notebook source # METADATA ******************** # META { # META "kernel_info": { # META "name": "synapse_pyspark" # META }, # META "dependencies": { # META "lakehouse": { # META "default_lakehouse": "47592d55-9a83-41a8-9b21-e1ef44264161", # META "default_lakehouse_name": "Example_LH", # META "default_lakehouse_workspace_id": "2190baad-a374-4114-addd-0dcf0533e69d" # META }, # META "environment": { # META "environmentId": "a277ea4a-e87f-8537-4ce0-39db11d4aade", # META "workspaceId": "00000000-0000-0000-0000-000000000000" # META } # META } # META } # CELL ******************** df = spark.sql("SELECT * FROM Example_LH.Table1 LIMIT 1000") display(df) # METADATA ******************** # META { # META "language": "python", # META "language_group": "synapse_pyspark" # META } ``` #### Advanced `find_replace` Parameterization Case **Case:** A Notebook is attached to a Lakehouse which resides in the same workspace. When deploying both the Lakehouse and the Notebook to a target environment (PPE/PROD/etc), the Workspace and Lakehouse GUIDs referenced in the Notebook must be updated to ensure it correctly points to the corresponding Lakehouse in the new environment. **Solution:** This approach uses `find_value` [**regex**](#find_value-regex)\*\* and [**dynamic variables**](#dynamic-replacement) to manage replacement. In the `find_replace` input in the `parameter.yml` file, the `is_regex` field is set to `"true"`, enabling fabric-cicd to find a string value within the _specified_ repository files that matches the provided regex pattern. This approach is particularly useful for replacing values that are not known until deployment time, such as item IDs. \*\*The regex pattern must include a capture group, defined using `()`, and the `find_value` must always match **group 1**. The value captured in this group will be dynamically replaced with the appropriate value for the deployed environment. parameter.yml file ```yaml find_replace: # lakehouse GUID matching group 1 of regex pattern to be replaced - find_value: \#\s*META\s+"default_lakehouse":\s*"([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})" replace_value: PPE: "$items.Lakehouse.Example_LH.$id" # PPE lakehouse GUID (dynamic) PROD: "$items.Lakehouse.Example_LH.$id" # PROD lakehouse GUID (dynamic) is_regex: "true" item_type: "Notebook" # filter on notebook files item_name: ["Hello World", "Goodbye World"] # filter on specific notebook files # workspace ID matching group 1 of regex pattern to be replaced - find_value: \#\s*META\s+"default_lakehouse_workspace_id":\s*"([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})" replace_value: PPE: "$workspace.$id" # PPE workspace ID (dynamic) PROD: "$workspace.$id" # PROD workspace ID (dynamic) is_regex: "true" file_path: # filter on notebook files with these paths - "/Hello World.Notebook/notebook-content.py" - "\\Goodbye World.Notebook\\notebook-content.py" ``` notebook-content.py file ```python # Fabric notebook source # METADATA ******************** # META { # META "kernel_info": { # META "name": "synapse_pyspark" # META }, # META "dependencies": { # META "lakehouse": { # META "default_lakehouse": "123e4567-e89b-12d3-a456-426614174000", # META "default_lakehouse_name": "Example_LH", # META "default_lakehouse_workspace_id": "8f5c0cec-a8ea-48cd-9da4-871dc2642f4c" # META }, # META "environment": { # META "environmentId": "a277ea4a-e87f-8537-4ce0-39db11d4aade", # META "workspaceId": "00000000-0000-0000-0000-000000000000" # META } # META } # META } # CELL ******************** df = spark.sql("SELECT * FROM Example_LH.Table1 LIMIT 1000") display(df) # METADATA ******************** # META { # META "language": "python", # META "language_group": "synapse_pyspark" # META } ``` #### TSQL Notebook with SQL Endpoint Parameterization Case **Case:** A TSQL Notebook is attached to a Lakehouse SQL Endpoint. The SQL Endpoint ID in the Notebook needs to be updated to point to the corresponding SQL Endpoint in the target environment. **Solution:** Use `$items.Lakehouse..$sqlendpointid` to dynamically retrieve the SQL Endpoint ID. parameter.yml file ```yaml find_replace: - find_value: "37dc8a41-dea9-465d-b528-3e95043b2356" replace_value: PPE: "$items.Lakehouse.Example_LH.$sqlendpointid" PROD: "$items.Lakehouse.Example_LH.$sqlendpointid" item_type: "Notebook" ``` notebook-content.py file ```python # META { # META "dependencies": { # META "lakehouse": { # META "default_lakehouse": "123e4567-e89b-12d3-a456-426614174000", # META "default_lakehouse_name": "Example_LH", # META "default_lakehouse_sql_endpoint": "37dc8a41-dea9-465d-b528-3e95043b2356" # META } # META } # META } ``` ### Data Pipelines #### `key_value_replace` Parameterization Case **Case:** A Data Pipeline is attached to data sources via the Connection Id. Connections are not deployed with fabric-cicd and therefore need to be parameterized. In the `pipeline-content.json` file, the SQL Server Connection Id `c517e095-ed87-4665-95fa-8cdb1e751fba`, must be replaced with the corresponding GUIDs of the SQL Server in the target environment (PPE/PROD/etc). **Solution:** This replacement is managed by the `find_key` input in the `parameter.yml` file where fabric-cicd finds every instance of the key within the _specified_ repository files and replaces it with the string for the deployed environment. parameter.yml file ```yaml key_value_replace: - find_key: $.properties.activities[?(@.name=="Copy Data")].typeProperties.source.datasetSettings.externalReferences.connection replace_value: PPE: "f47ac10b-58cc-4372-a567-0e02b2c3d479" # PPE SQL Connection GUID PROD: "9b2e5f4c-8d3a-4f1b-9c3e-2d5b6e4a7f8c" # PROD SQL Connection GUID item_type: "DataPipeline" # filter on Data Pipelines files item_name: "Example Pipeline" # filter on specific Data Pipelines files ``` pipeline-content.json file ```json { "properties": { "activities": [ { "name": "Copy Data", "type": "Copy", "dependsOn": [], "policy": { "timeout": "0.12:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "typeProperties": { "source": { "type": "AzureSqlSource", "queryTimeout": "02:00:00", "partitionOption": "None", "datasetSettings": { "annotations": [], "type": "AzureSqlTable", "schema": [], "typeProperties": { "schema": "Dataprod", "table": "DIM_Calendar", "database": "unified" }, "externalReferences": { "connection": "c517e095-ed87-4665-95fa-8cdb1e751fba" } } }, "sink": { "type": "LakehouseTableSink", "tableActionOption": "Append", "datasetSettings": { "annotations": [], "linkedService": { "name": "Unified", "properties": { "annotations": [], "type": "Lakehouse", "typeProperties": { "workspaceId": "2d2e0ae2-9505-4f0c-ab42-e76cc11fb07d", "artifactId": "31dd665e-95f3-4575-9f46-70ea5903d89b", "rootFolder": "Tables" } } }, "type": "LakehouseTable", "schema": [], "typeProperties": { "schema": "Dataprod", "table": "DIM_Calendar" } } }, "enableStaging": false, "translator": { "type": "TabularTranslator", "typeConversion": true, "typeConversionSettings": { "allowDataTruncation": true, "treatBooleanAsNumber": false } } } } ] } } ``` ### Schedules #### `key_value_replace` Parameterization Case **Case:** Items with schedules need to have their execution settings parameterized across environments. The `enabled` field in `.schedules` files typically needs different values for different environments (e.g., disabled in test environments, enabled in production). **Solution:** In the `.schedules` file, the `enabled` field for Execute jobs must be replaced with environment-specific boolean values. This replacement is managed by the `key_value_replace` input in the `parameter.yml` file where fabric-cicd finds the JSONPath expression within the specified repository files and replaces it with the appropriate value for the deployed environment. parameter.yml file ```yaml key_value_replace: - find_key: $.schedules[?(@.jobType=="Execute")].enabled replace_value: PPE: false PROD: true file_path: "**/.schedules" ``` .schedules file ```json { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/schedules/1.0.0/schema.json", "schedules": [ { "enabled": true, "jobType": "Execute", "configuration": { "type": "Cron", "startDateTime": "2025-07-01T12:00:00", "endDateTime": "2029-07-01T12:00:00", "localTimeZoneId": "Pacific Standard Time", "interval": 15 } } ] } ``` **Note:** The `.schedules` file contains JSON content but does not have a `.json` file extension. The fabric-cicd library automatically detects and processes files with valid JSON content regardless of their file extension, making this parameterization work seamlessly. ### Environments #### `spark_pool` Parameterization Case **Case:** An Environment is attached to a Capacity level Custom Pool. The `instance_pool_id` in `Sparkcompute.yml` is workspace-specific and will differ between environments. The Spark Pool needs to be parameterized so fabric-cicd can resolve the correct pool in the target workspace. **Note:** Defining different names per environment is supported in the `parameter.yml` file. In the `Sparkcompute.yml` file, the referenced instance_pool_id `72c68dbc-0775-4d59-909d-a47896f4573b` points to a capacity custom pool named `CapacityPool_Large` of pool type `Capacity` for the `PROD` environment. **Solution:** This replacement is managed by the `spark_pool` input in the `parameter.yml` file where fabric-cicd finds every instance of the `instance_pool_id` and replaces it with the pool type and pool name for the _specified_ environment file. parameter.yml file ```yaml spark_pool: - instance_pool_id: "72c68dbc-0775-4d59-909d-a47896f4573b" # spark_pool_instance_id to be replaced replace_value: PPE: type: "Capacity" # target spark pool type, only supports Capacity or Workspace name: "CapacityPool_Medium" # target spark pool name PROD: type: "Capacity" # target spark pool type, only supports Capacity or Workspace name: "CapacityPool_Large" # target spark pool name item_name: "World" # filter on environment file for environment named "World" ``` Sparkcompute.yml ```yaml enable_native_execution_engine: false instance_pool_id: 72c68dbc-0775-4d59-909d-a47896f4573b driver_cores: 16 driver_memory: 112g executor_cores: 16 executor_memory: 112g dynamic_executor_allocation: enabled: false min_executors: 31 max_executors: 31 runtime_version: 1.3 ``` ### Dataflows Dataflows can have different kinds of Fabric sources and destinations that need to be parameterized, depending on the scenario. #### Parameterization Overview Take a Lakehouse source/destination as an example, the Lakehouse is connected to a Dataflow in the following ways: 1. Connection Id in the `queryMetadata.json` file: - Connections are not deployed with fabric-cicd and therefore need to be parameterized. 2. Workspace and item IDs in the `mashup.pq` file: - Source and/or destination item references, such as a Dataflow (source only\*\*), Lakehouse, Warehouse, etc. appear in the `mashup.pq` file and need to be parameterized to ensure proper deployment across environments. **\*\*Note:** A Dataflow that sources from another Dataflow introduces a dependency that may require a specific order of deploying (source first then dependent). A Dataflow is referenced by the item ID in the workspace and the actual workspace ID, this makes re-pointing more complex (see parameterization guidance below). #### Parameterization Guidance Connections must be parameterized in addition to item references. Scenarios When Deploying a Dataflow that contains a source Dataflow reference: 1. Source Dataflow exists in the **same workspace** as the dependent Dataflow: - The source Dataflow must be deployed BEFORE the dependent Dataflow (especially during first time deployment). - To handle this dependency correctly and prevent deployment errors, set up the `find_replace` parameter with the following requirements (incorrect setup may introduce failure during Dataflow deployment): - Set `find_value` to match the `dataflowId` GUID referenced in the `mashup.pq` file (literal string or [regex](#find_value-regex)). - Set `replace_value` to the variable `$items.Dataflow..$id`. **Important:** Make sure the **item type** is `"Dataflow"` and the **item name** matches the source Dataflow name in the repository directory exactly (case sensitive, include any spaces). - File filters are optional but recommended when using a regex pattern for `find_value`. - **You don't need to parameterize the source Dataflow workspace ID here** as the library automatically handles this replacement when you use the items variable in _this_ Dataflow scenario. - **How this works:** This parameterization approach ensures correct deployment of interdependent Dataflows while automatically updating references to point to the newly deployed Dataflow in the target workspace. - Example parameter input: ```yaml find_replace: # The ID of the source Dataflow referenced in mashup.pq - find_value: "0187104d-7a35-4abe-a2ca-a241ec81c8f1" # Type = Dataflow and Name = , Attribute = id replace_value: PPE: "$items.Dataflow.Source Dataflow.$id" PROD: "$items.Dataflow.Source Dataflow.$id" # Optional fields: file_path: - "\\Referencing Dataflow.Dataflow\\mashup.pq" ``` 2. Source Dataflow exists in a **different workspace** from the dependent Dataflow: - When the source Dataflow exists in a different workspace, deployment order doesn't matter. - To re-point the source Dataflow from one workspace to another workspace, you can parameterize using the `find_replace` parameter. The Dataflow ID AND Workspace ID of the source Dataflow both need to be parameterized. - **Note:** dynamic replacement for item ID and workspace ID will NOT work here since the source Dataflow does not exist in the _repository directory_. Scenarios When Deploying a Dataflow that contains other Fabric items (e.g., Lakehouse, Warehouse, etc.) references: 1. Source and/or destination item exists in the **same workspace** as the dependent Dataflow: - Use the `find_replace` parameter to update references so they point to the corresponding items in the target workspace. - You need to parameterize both the item ID and workspace ID found in the `mashup.pq` file. - Best practices for Dataflow parameterization: - Use a [regex](#find_value-regex) for the `find_value` to avoid hardcoding GUIDs and simplify maintenance - Use [dynamic replacement](#dynamic-replacement) to eliminate multi-phase deployments - Adding file filters to target specific Dataflow files provides more precise control. 2. Source/destination item exists in a **different workspace** from the dependent Dataflow: - Use the `find_replace` parameter to update references so they point to items in the different workspace. - Parameterize both the item ID and workspace ID found in the `mashup.pq` file. - Use a regex pattern for the `find_value` to avoid hardcoding GUIDs and simplify maintenance. - **Note:** dynamic replacement won't work in this scenario - it only works for items in the same workspace as the Dataflow. - Adding file filters helps target specific Dataflow files for more precise control. #### Advanced `find_replace` Parameterization Case **Case:** A Dataflow points to a destination Lakehouse. The Lakehouse exists in the same workspace as the Dataflow. In the `mashup.pq` file, the following GUIDs need to be replaced: - The workspaceId `e6a8c59f-4b27-48d1-ae03-7f92b1c6458d` with the target workspace Id. - The lakehouseId `3d72f90e-61b5-42a8-9c7e-b085d4e31fa2` with the corresponding Id of the Lakehouse in the target environment (PPE/PROD/etc). **Solution:** These replacements are managed using a regex pattern as input for the `find_value` in the `parameter.yml` file, which finds the matching value in the _specified_ repository files and replaces it with the dynamically retrieved workspace or item Id of the target environment. **Note:** While Connection IDs are shown in this example, they are not the main focus. Connection parameterization may vary depending on your specific scenario. parameter.yml file ```yaml find_replace: # Lakehouse workspace ID regex - matches the workspaceId GUID - find_value: Navigation_1\s*=\s*Pattern\{\[workspaceId\s*=\s*"([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})"\]\} replace_value: PPE: "$workspace.$id" # PPE workspace ID (dynamic) PROD: "$workspace.$id" is_regex: "true" # Activate find_value regex matching file_path: "/Sample Dataflow.Dataflow/mashup.pq" # Lakehouse ID regex - matches the lakehouseId GUID - find_value: Navigation_2\s*=\s*Navigation_1\{\[lakehouseId\s*=\s*"([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})"\]\} replace_value: PPE: "$items.Lakehouse.Sample_LH.$id" # Sample_LH Lakehouse ID in PPE (dynamic) PROD: "$items.Lakehouse.Sample_LH.$id" is_regex: "true" # Activate find_value regex matching file_path: "/Sample Dataflow.Dataflow/mashup.pq" # Connection ID - Cluster ID - find_value: "8e4f92a7-3c18-49d5-b6d0-7f2e591ca4e8" replace_value: PPE: "76a8f5c3-e4b2-48d1-9c7f-382d69a5e7b0" # PPE Cluster ID PROD: "f297e14d-6c83-42a5-b718-59d40e3f8c2d" # PROD Cluster ID file_path: "/Sample Dataflow.Dataflow/queryMetadata.json" # Connection ID - Datasource ID - find_value: "d12c5f7b-90a3-47e6-8d2c-3fb59e01d47a" replace_value: PPE: "25b9a417-3d8e-4f62-901c-75de6ba84f35" # PPE Datasource ID PROD: "cb718d96-5ae2-47fc-8b93-1d24c0f5e8a7" # PROD Datasource ID file_path: "/Sample Dataflow.Dataflow/queryMetadata.json" ``` queryMetadata.json file ```json { "formatVersion": "202502", "computeEngineSettings": {}, "name": "Sample Dataflow", "queryGroups": [], "documentLocale": "en-US", "queriesMetadata": { "Table": { "queryId": "ba67667b-14c0-4536-a92d-feafc73baa4b", "queryName": "Table", "loadEnabled": false }, "Table_DataDestination": { "queryId": "a157a378-b510-4d95-bb82-5a7c80df8b4c", "queryName": "Table_DataDestination", "isHidden": true, "loadEnabled": false } }, "connections": [ { "path": "Lakehouse", "kind": "Lakehouse", "connectionId": "{\"ClusterId\":\"8e4f92a7-3c18-49d5-b6d0-7f2e591ca4e8\",\"DatasourceId\":\"d12c5f7b-90a3-47e6-8d2c-3fb59e01d47a\"}" } ] } ``` mashup.pq file ```pq [StagingDefinition = [Kind = "FastCopy"]] section Section1; [DataDestinations = {[Definition = [Kind = "Reference", QueryName = "Table_DataDestination", IsNewTarget = true], Settings = [Kind = "Automatic", TypeSettings = [Kind = "Table"]]]}] shared Table = let Source = Table.FromRows(Json.Document(Binary.Decompress(Binary.FromText("i45WckksSUzLyS9X0lEyBGKP1JycfKVYHYhEQGZBak5mXipQwgiIw/OLclLAkn75JalJ+fnZQEFjmC4FhHRwam5iXklmsm9+SmoOUN4EiMFsBVTzoRabArGLG0x/LAA=", BinaryEncoding.Base64), Compression.Deflate)), let _t = ((type nullable text) meta [Serialized.Text = true]) in type table [Item = _t, Id = _t, Name = _t]), #"Changed column type" = Table.TransformColumnTypes(Source, {{"Item", type text}, {"Id", Int64.Type}, {"Name", type text}}), #"Added custom" = Table.TransformColumnTypes(Table.AddColumn(#"Changed column type", "IsDataflow", each if [Item] = "Dataflow" then true else false), {{"IsDataflow", type logical}}), #"Added custom 1" = Table.TransformColumnTypes(Table.AddColumn(#"Added custom", "ContainsHello", each if Text.Contains([Name], "Hello") then 1 else 0), {{"ContainsHello", Int64.Type}}) in #"Added custom 1"; shared Table_DataDestination = let Pattern = Lakehouse.Contents([CreateNavigationProperties = false, EnableFolding = false]), Navigation_1 = Pattern{[workspaceId = "e6a8c59f-4b27-48d1-ae03-7f92b1c6458d"]}[Data], Navigation_2 = Navigation_1{[lakehouseId = "3d72f90e-61b5-42a8-9c7e-b085d4e31fa2"]}[Data], TableNavigation = Navigation_2{[Id = "Items", ItemKind = "Table"]}?[Data]? in TableNavigation; ``` ### Reports Reports can reference Semantic Models in two ways: `byPath` (relative path to a model in the same repository) or `byConnection` (connection string to a model in Power BI service). #### Parameterization Overview **`byPath` - No Parameterization Needed** When a Report uses `byPath` to reference a Semantic Model in the same repository, the library automatically converts it to `byConnection` format during deployment. The Semantic Model must exist in the repository for this to work. **`byConnection` - Requires Parameterization** When Reports and Semantic Models are deployed separately (e.g., models first, then reports), or when Reports need to connect to models in Power BI Online service, use `byConnection` with parameterization to rebind reports to different semantic models across environments. This approach supports both same-workspace and cross-workspace binding scenarios. **Case:** A Report uses `byConnection` to reference a Semantic Model deployed to Power BI Online. The connection string contains environment-specific values (workspace ID, semantic model name, and semantic model ID) that must be updated for each target environment (PPE, PROD, etc.). The semantic model can be in the same workspace as the report or in a different workspace. **Note:** This case does not apply to the `semantic_model_binding` parameter. **Solution:** Use `find_replace` or `key_value_replace` in the `parameter.yml` file to parameterize the connection string components. #### `find_replace` Parameterization Case This approach replaces individual parts of the connection string (workspace ID, model name, model ID) with environment-specific values. This enables granular control over each component in the connection string and allows the option to apply dynamic variables where needed. **Note:** The examples below use placeholder values (e.g., `MyReport`, `YourSemanticModelName`). Replace these with your actual report and semantic model names. For a working example, see `sample/workspace/parameter.yml` which references the `ByConnection.Report` and `ABC.SemanticModel` items. parameter.yml file ```yaml find_replace: # Replace workspace ID in connection string - find_value: "dev-workspace-id" replace_value: PPE: "$workspace.$id" # PPE workspace ID PROD: "$workspace.$id" # PROD workspace ID item_type: "Report" file_path: "/MyReport.Report/definition.pbir" # Replace semantic model name in connection string - find_value: "dev-semantic-model" replace_value: PPE: "ppe-semantic-model-name" PROD: "prod-semantic-model-name" item_type: "Report" file_path: "/MyReport.Report/definition.pbir" # Replace semantic model ID in connection string with dynamic replacement - find_value: "00000000-0000-0000-0000-000000000000" replace_value: PPE: "$items.SemanticModel.YourSemanticModelName.$id" PROD: "$items.SemanticModel.YourSemanticModelName.$id" item_type: "Report" file_path: "/MyReport.Report/definition.pbir" ``` definition.pbir ```json { "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/report/definitionProperties/2.0.0/schema.json", "version": "4.0", "datasetReference": { "byConnection": { "connectionString": "Data Source=powerbi://api.powerbi.com/v1.0/myorg/dev-workspace-id;initial catalog=dev-semantic-model;access mode=readonly;integrated security=ClaimsToken;semanticmodelid=00000000-0000-0000-0000-000000000000" } } } ``` #### `key_value_replace` Parameterization Case This approach replaces the entire connection string with environment-specific values. This simplifies the parameter configuration, however, dynamic variables are not supported in this example as they cannot be embedded within a larger string value. parameter.yml file ```yaml key_value_replace: - find_key: $.datasetReference.byConnection.connectionString replace_value: PPE: "Data Source=powerbi://api.powerbi.com/v1.0/myorg/ppe-workspace-guid;initial catalog=ppe-semantic-model;access mode=readonly;integrated security=ClaimsToken;semanticmodelid=ppe-model-guid" PROD: "Data Source=powerbi://api.powerbi.com/v1.0/myorg/prod-workspace-guid;initial catalog=prod-semantic-model;access mode=readonly;integrated security=ClaimsToken;semanticmodelid=prod-model-guid" item_type: "Report" item_name: "MyReport" ``` definition.pbir ```json { "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/report/definitionProperties/2.0.0/schema.json", "version": "4.0", "datasetReference": { "byConnection": { "connectionString": "Data Source=powerbi://api.powerbi.com/v1.0/myorg/dev-workspace-id;initial catalog=dev-semantic-model;access mode=readonly;integrated security=ClaimsToken;semanticmodelid=00000000-0000-0000-0000-000000000000" } } } ``` ================================================ FILE: docs/how_to/troubleshooting.md ================================================ # Troubleshooting This guide provides comprehensive debugging and troubleshooting resources for both users deploying with fabric-cicd and contributors developing within the repository. ## Debugging Deployments ### Enable Debug Logging fabric-cicd includes a debug logging feature that provides detailed visibility into all operations, including API calls made during deployment. **Default Behavior:** - Without debug logging enabled, fabric-cicd displays only high-level progress messages, warnings, and errors - The `fabric_cicd.error.log` file will contain the same lines printed to the console along with stack traces for any errors **Enabling Debug Logging:** To enable debug logging, add the following to your deployment script: ```python from fabric_cicd import change_log_level # Enable debug logging (call before other fabric-cicd operations) change_log_level() ``` When debug logging is enabled, all API calls are logged with detailed request/response information, and additional context about internal operations is displayed. Both the console and the `fabric_cicd.error.log` file will contain the detailed information. **Important:** Always enable debug logging when troubleshooting deployment issues. The additional output helps identify whether problems originate from API calls, authentication, or configuration. See [Understanding Error Logs](#understanding-error-logs) for details on interpreting log output. ### Testing Deployments Locally Before running deployments via CI/CD pipelines, users can test the deployment workflow locally by running the provided debug scripts. This helps with: - Validating configuration changes without affecting production - Testing parameter file configurations - Debugging deployment issues - Verifying authentication and permissions fabric-cicd includes several debug scripts in the `devtools/` directory that allow users to run deployments against real workspaces in a controlled environment. See [Debug Scripts](#debug-scripts) for detailed information on: - `debug_local.py` or `debug_local config.py` - Test full deployment workflows - `debug_parameterization.py` - Validate parameter files without deploying - `debug_api.py` - Test Fabric REST API calls directly - `debug_trace_deployment.py` - Perform and end-to-end deployment against a Fabric Workspace and capture HTTP Traces to be used for Integration Tests **Tip:** Using these scripts locally can catch configuration errors early, saving time in your CI/CD pipeline. ### Sample Workspace Directory fabric-cicd includes the `sample/workspace/` directory that demonstrates the recommended repository structure for Fabric item source control files. It contains sample items of various supported item types (e.g., Environment, Notebook, Data Pipeline, etc.). **Repository Directory Structure:** ``` sample/workspace/ ├── Sample Pipeline.DataPipeline/ │ ├── .platform │ └── pipeline-content.json ├── Sample_Notebook.Notebook/ │ ├── .platform │ └── notebook-content.py ... └── parameter.yml ``` Each item folder follows the naming convention `ItemName.ItemType/` and contains: - `.platform` file which contains the item metadata - Item definition files (e.g., `pipeline-content.json`, `notebook-content.py`) **Using the Sample:** Use this sample structure as a template for organizing your Fabric items. To test deployments with the items found in the sample workspace, set `repository_directory = "sample/workspace"` in `debug_local.py` or in `config.yml` when running `debug_local config.py`. ### Understanding Error Logs When running a deployment, fabric-cicd automatically creates a `fabric_cicd.error.log` file in your working directory. The level of detail captured depends on whether [debug logging is enabled](#enable-debug-logging). **Tip:** Always enable debug logging when troubleshooting deployment issues to capture full API traces in the log file. #### Accessing API Traces When an error occurs during deployment, the console will display: ``` Error: [Brief error message] See /path/to/fabric_cicd.error.log for full details. ``` Open the `fabric_cicd.error.log` file to view: 1. **Request Details**: The exact API endpoint called, HTTP method, and request body 2. **Response Details**: Status code, response headers, and complete response body 3. **Timing Information**: When the call was made 4. **Stack Trace**: The complete call stack leading to the error 5. **Additional Logs**: Information on internal operations that occurred during deployment This information is critical for determining if issues are caused by: - API failures or service issues - Authentication/authorization problems - Invalid request payloads - Network connectivity issues #### Example Error Log Entry ``` 2024-01-06 10:30:45 - ERROR - fabric_cicd.api - API call failed Request: POST https://api.fabric.microsoft.com/v1/workspaces/{workspace_id}/items Headers: {'Authorization': '***', 'Content-Type': 'application/json'} Body: {"displayName": "MyNotebook", "type": "Notebook", ...} Response: 400 Bad Request Body: {"error": {"code": "InvalidRequest", "message": "Item name contains invalid characters"}} Traceback (most recent call last): File "fabric_cicd/publish.py", line 123, in publish_item response = api.create_item(...) ... ``` ### Common Issues and Solutions #### Authentication Failures **Symptom**: Errors mentioning "authentication failed" or "401 Unauthorized" **Solution**: 1. Explicit authentication is **required** as the `DefaultAzureCredential` fallback has been removed. `token_credential` is now a required parameter. fabric-cicd accepts any [`TokenCredential`](https://learn.microsoft.com/en-us/python/api/azure-core/azure.core.credentials.tokencredential) — choose the appropriate one for your scenario: - Local development: `AzureCliCredential` (requires `az login`) or `AzurePowerShellCredential` (requires `Connect-AzAccount`) - CI/CD pipelines with platform auth: `AzureCliCredential` or `AzurePowerShellCredential` (requires a prior login step in the workflow, e.g., `azure/login` or AzCLI task) - CI/CD pipelines with OIDC / workload identity federation: `WorkloadIdentityCredential` (secretless; recommended for GitHub Actions and Azure DevOps with federated credentials) - CI/CD pipelines with service principals: `ClientSecretCredential` (requires client ID, secret, and tenant ID) - CI/CD pipelines with managed identity: `ManagedIdentityCredential` (requires Azure-hosted self-hosted runners) - Fabric Notebooks: Provide an explicit credential. See Authentication Examples for details. 2. Verify authentication setup: ```bash az login ``` or ```powershell Connect-AzAccount ``` 3. Check permissions: ensure your account has appropriate permissions on the target workspace 4. For Service Principal authentication: verify client ID, secret, and tenant ID are correct 5. See detailed examples: refer to [authentication examples](../example/authentication.md) for platform-specific implementation guidance #### Item Deployment Failures **Symptom**: Specific items fail to deploy while others succeed **Solution**: 1. Enable debug logging to see the exact API error 2. Check `fabric_cicd.error.log` for detailed API response 3. Verify the item definition files exist and are properly formatted 4. Check if the item type is included in your `item_type_in_scope` list 5. Ensure item dependencies exist (e.g., a Data Pipeline referencing a Notebook must be deployed along with the Notebook) 6. If deleting and recreating an item with the same name, wait 5 minutes between operations due to Fabric API item name reservation #### Parameter Substitution Issues **Symptom**: Deployed items contain literal find value instead of the proper replace value **Solution**: 1. Verify your `parameter.yml` file is in the correct location (repository directory by default) 2. Check that find values in your files exactly match those in `parameter.yml` 3. Ensure the environment name matches between your script and `parameter.yml` 4. Validate the find value regex and/or dynamic replacement variables in `parameter.yml` 5. Use the [debug_parameterization.py](#debug_parameterizationpy) script to validate parameter files #### Private Link Connection Failures **Symptom**: API calls fail with connection errors when deploying to a workspace with "Allow connections only from workspace level private links" enabled. **Solution**: Call `configure_fabric_fqdn` before initializing `FabricWorkspace`: ```python from fabric_cicd import configure_fabric_fqdn, FabricWorkspace configure_fabric_fqdn(workspace_id) workspace = FabricWorkspace(workspace_id=workspace_id, ...) ``` #### API Rate Limiting **Symptom**: Deployments fail with "429 Too Many Requests" errors **Solution**: 1. Consider deploying in smaller batches 2. Check `fabric_cicd.error.log` for retry-after headers in API responses ### Debug Scripts The `devtools/` directory contains pre-built scripts to help test and validate deployments, parameter files, and Fabric REST APIs locally. These scripts already exist in the repository - you just need to configure them for your scenario. #### debug_local.py **Purpose**: Test full deployment workflows locally against a Microsoft Fabric workspace. **Key Configuration Options**: | Configuration | Description | Required | | ---------------------- | ---------------------------------------------------------------- | -------- | | `workspace_id` | Target Fabric workspace ID | Yes | | `environment` | Target environment (used for parameterization) | No | | `repository_directory` | Path to Fabric workspace items files (absolute or relative path) | Yes | | `item_type_in_scope` | Specific item types to deploy (defaults to all supported types) | No | | `token_credential` | Explicit credential method (`AzureCliCredential`, etc.) | Yes | **Quick Start**: 1. Open `devtools/debug_local.py` 2. Set `workspace_id`, `environment`, and `repository_directory` at the top 3. Uncomment `change_log_level()` to enable debug logging 4. Add necessary [feature flags](optional_feature.md#feature-flags) required for deployment 5. Uncomment `publish_all_items(target_workspace)` and/or `unpublish_all_orphan_items(target_workspace)` to test deployment 6. Run: `uv run python devtools/debug_local.py` **Common Configurations**: ```python # Enable debug logging change_log_level() # Add feature flag(s) append_feature_flag("enable_shortcut_publish") # Use sample workspace for testing repository_directory = "sample/workspace" # Deploy only specific item types item_type_in_scope = ["Environment", "Notebook", "DataPipeline"] # Authentication examples - choose one: # For local development with Azure CLI from azure.identity import AzureCliCredential token_credential = AzureCliCredential() # For local development with Azure PowerShell from azure.identity import AzurePowerShellCredential token_credential = AzurePowerShellCredential() # For CI/CD with Service Principal from azure.identity import ClientSecretCredential token_credential = ClientSecretCredential( client_id="your-client-id", client_secret="your-client-secret", tenant_id="your-tenant-id" ) # For Azure-hosted runners with Managed Identity from azure.identity import ManagedIdentityCredential token_credential = ManagedIdentityCredential() # Override constant value constants.DEFAULT_API_ROOT_URL = "https://api.fabric.microsoft.com" ``` #### debug_local config.py **Purpose**: Test configuration-based deployment workflows using a `config.yml` file. **Key Configuration Options**: | Configuration | Description | Required | | ------------------ | ----------------------------------------------------------------------------------- | -------- | | `config_file` | Path to your `config.yml` file | Yes | | `token_credential` | Explicit credential method (`AzureCliCredential`, etc.) | Yes | | `environment` | Target environment (used for parameterization and environment-based configurations) | No | | `config_override` | Dictionary to override configuration values within `config.yml` | No | **Quick Start**: 1. Open `devtools/debug_local config.py` 2. Set `config_file` path and `environment` (can use the sample `config.yml` file found in `sample/workspace`) 3. Uncomment `change_log_level()` to enable debug logging 4. Ensure required feature flags are enabled (already set in script) 5. Run: `uv run python "devtools/debug_local config.py"` See [configuration deployment](config_deployment.md) for details on creating `config.yml`. #### debug_parameterization.py **Purpose**: Validate parameter file without deploying items - useful for catching parameterization errors early. **Key Configuration Options**: | Configuration | Description | Required | | ---------------------- | ------------------------------------------------------------------------------------ | -------- | | `repository_directory` | Path to Fabric workspace items files and `parameter.yml` file (default location) | Yes | | `environment` | Target environment (used for parameterization) | No | | `item_type_in_scope` | Item types to validate (defaults to all) | No | | `parameter_file_name` | Alternate parameter file name within repository directory (default: `parameter.yml`) | No | | `parameter_file_path` | Alternate location of parameter file | No | **Quick Start**: 1. Open `devtools/debug_parameterization.py` 2. Set `repository_directory` and `environment` (the `parameter.yml` file is located in the repository directory in this case) 3. Uncomment `change_log_level()` to view all the validation steps 4. Run: `uv run python devtools/debug_parameterization.py` See [parameterization](parameterization.md#parameter-file-validation) for more information. #### debug_api.py **Purpose**: Test Fabric REST API calls directly without going through full deployment workflows. **Key Configuration Options**: | Configuration | Description | Required | | ------------------ | ------------------------------------------------------------------- | -------- | | `token_credential` | Explicit credential method (`AzureCliCredential`, etc.) | Yes | | `api_url` | Full API endpoint URL | Yes | | `method` | HTTP method (GET, POST, DELETE, PATCH) | Yes | | `body` | Request payload (for POST/PATCH) | Varies | | other | View `invoke()` in `FabricEndpoint` class for additional parameters | No | **Quick Start**: 1. Open `devtools/debug_api.py` 2. Configure the API endpoint, method, body (if any) 3. Uncomment `change_log_level()` to view API request/response details 4. Run: `uv run python devtools/debug_api.py` #### debug_trace_deployment.py **Purpose**: Debug the public APIs called in `publish_all_items()` workflow with breakpoints using VS Code's debugger. **Quick Start**: 1. Open `devtools/debug_trace_deployment.py` 2. Set breakpoint(s) in the code - e.g. prior to `publish_all_items` 3. Update `.vscode/launch.json` with your workspace ID in `FABRIC_WORKSPACE_ID` 4. Press **F5** → Select "Debug: Publish All Items" ## Getting Help If you're still experiencing issues after following this guide: 1. **Enable debug logging** and capture the complete error log 2. **Check existing issues** on [GitHub](https://github.com/microsoft/fabric-cicd/issues) 3. **Create a new issue** using the appropriate template: - [Bug Report](https://github.com/microsoft/fabric-cicd/issues/new?template=1-bug.yml) - [Question](https://github.com/microsoft/fabric-cicd/issues/new?template=4-question.yml) ## Additional Resources - [Authentication Examples](../example/authentication.md) - Comprehensive authentication implementation examples - [Contribution Guide](https://github.com/microsoft/fabric-cicd/blob/main/CONTRIBUTING.md) - Setup instructions and PR requirements - [Feature Flags](optional_feature.md#feature-flags) - Available feature flags for advanced scenarios - [Getting Started](getting_started.md) - Basic installation and authentication - [Microsoft Fabric API Documentation](https://learn.microsoft.com/en-us/rest/api/fabric/) - Official API reference ================================================ FILE: docs/index.md ================================================ fabric-cicd is a Python library designed for use with [Microsoft Fabric](https://learn.microsoft.com/en-us/fabric/) workspaces. This library supports code-first Continuous Integration / Continuous Deployment (CI/CD) automations to seamlessly integrate Source Controlled workspaces into a deployment framework. The goal is to assist CI/CD developers who prefer not to interact directly with the Microsoft Fabric APIs. ## Base Expectations - Full deployment every time, without considering commit diffs - Deploys into the tenant of the executing identity - Only supports items that have Source Control, and Public Create/Update APIs ## Supported Item Types ## Installation **Requirements**: Python to To install fabric-cicd, run: ```bash pip install fabric-cicd ``` ## Basic Example ```python from azure.identity import AzureCliCredential from fabric_cicd import FabricWorkspace, publish_all_items, unpublish_all_orphan_items token_credential = AzureCliCredential() # Initialize the FabricWorkspace object with the required parameters target_workspace = FabricWorkspace( workspace_id="your-workspace-id", environment="your-target-environment", repository_directory="your-repository-directory", item_type_in_scope=["Notebook", "DataPipeline", "Environment"], token_credential=token_credential, # or any other TokenCredential ) # Publish all items defined in item_type_in_scope publish_all_items(target_workspace) # Unpublish all items defined in item_type_in_scope not found in repository unpublish_all_orphan_items(target_workspace) ``` > **Notes:** > > - All parameters for `FabricWorkspace` must be passed as keyword arguments. > - The `environment` parameter is required for parameter replacement to work properly. It must match one of the environment keys defined in your `parameter.yml` file (e.g., "PPE", "PROD", "DEV"). If you don't need parameter replacement, you can omit this parameter. ================================================ FILE: mkdocs.yml ================================================ site_name: fabric-cicd repo_name: microsoft/fabric-cicd repo_url: https://github.com/microsoft/fabric-cicd site_url: https://microsoft.github.io/fabric-cicd/ remote_branch: gh-pages remote_name: origin nav: - Home: index.md - How To: - How To: how_to/index.md - Getting Started: how_to/getting_started.md - Item Types: how_to/item_types.md - Configuration Deployment: how_to/config_deployment.md - Parameterization: how_to/parameterization.md - Optional Features: how_to/optional_feature.md - Troubleshooting: how_to/troubleshooting.md - Examples: - Examples: example/index.md - Authentication: example/authentication.md - Deployment Variables: example/deployment_variable.md - Release Pipelines: example/release_pipeline.md - Code Reference: code_reference.md - Changelog: changelog.md - About: about.md theme: name: material custom_dir: docs/config/overrides icon: logo: fontawesome/solid/code favicon: config/assets/favicon.ico font: text: Roboto code: Roboto Mono features: - content.code.copy - content.tooltips - navigation.expand - navigation.indexes - navigation.sections - navigation.tabs - navigation.tabs.sticky - navigation.top - search.highlight - search.suggest - toc.follow - toc.integrate - announce.dismiss language: en palette: scheme: fabric extra_css: - config/stylesheets/extra.css hooks: - docs/config/pre-build/update_item_types.py - docs/config/pre-build/section_toc.py - docs/config/pre-build/update_python_version.py plugins: - search: separator: '[\s\u200b\-_,:!=\[\]()"`/]+|\.(?!\d)|&[lg]t;|(?!\b)(?=[A-Z][a-z])' - minify: minify_html: true - include-markdown - mkdocstrings: handlers: python: paths: ["src"] options: docstring_style: google docstring_options: ignore_init_summary: true summary: modules: false functions: true classes: true show_source: false show_root_full_path: false separate_signature: true show_signature_annotations: true signature_crossrefs: true filters: ["!^_", "^__init__$"] merge_init_into_class: true show_symbol_type_heading: true show_symbol_type_toc: true line_length: 80 extra: version: provider: mike default: latest alias: true social: - icon: fontawesome/brands/github link: https://github.com/microsoft/fabric-cicd generator: false analytics: feedback: title: Was this page helpful? ratings: - icon: material/thumb-up-outline name: This page was helpful data: 1 note: Thanks for your feedback! - icon: material/thumb-down-outline name: This page could be improved data: 0 note: >- Thanks for your feedback! Help us improve this page by using our feedback form. markdown_extensions: - abbr - md_in_html - toc: permalink: true toc_depth: 2 - admonition - pymdownx.highlight: anchor_linenums: true line_spans: __span pygments_lang_class: true - pymdownx.inlinehilite - pymdownx.snippets - pymdownx.superfences - pymdownx.tabbed: alternate_style: true - pymdownx.superfences copyright: Copyright © Microsoft Corporation ================================================ FILE: pyproject.toml ================================================ [project] name = "fabric-cicd" authors = [{ name = "Microsoft Corporation" }] description = "Microsoft Fabric CI/CD" readme = "README.md" requires-python = ">=3.9,<3.14" license = "MIT" license-files = ["LICENSE"] dynamic = ["version"] classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", ] dependencies = [ "azure-identity>=1.25.0", "dpath>=2.2.0", "filetype>=1.2.0", "jsonpath-ng>=1.8.0", "pyyaml>=6.0.2", "requests>=2.32.3", ] [project.urls] Repository = "https://github.com/microsoft/fabric-cicd.git" Changelog = "https://github.com/microsoft/fabric-cicd/blob/main/docs/changelog.md" [dependency-groups] dev = [ "coverage>=7.6.10", "gitpython>=3.1.44", "mike>=2.1.3", "mkdocs-include-markdown-plugin>=7.1.2", "mkdocs-material>=9.6.5", "mkdocs-minify-plugin>=0.8.0", "mkdocstrings-python>=1.13.0", "pygments>=2.18.0,<2.20.0", "pytest>=8.3.4", "pytest-mock>=3.14.0", "ruff>=0.9.5", ] [build-system] requires = ["setuptools>=61.0"] build-backend = "setuptools.build_meta" [tool.setuptools] dynamic.version = { attr = "fabric_cicd.constants.VERSION" } packages.find.where = ["src"] [tool.uv] package = true python-preference = "only-managed" [tool.pytest.ini_options] addopts = "-v --tb=short" testpaths = ["tests"] pythonpath = ["src"] [tool.coverage.run] source = ["src"] omit = ["tests/*"] ================================================ FILE: ruff.toml ================================================ line-length = 120 exclude = ["sample/*", "docs/*"] [lint] # https://docs.astral.sh/ruff/rules/ select = [ "A", # flake8-builtins "ANN", # flake8-annotations-complexity "ARG", # flake8-unused-arguments "B", # flake8-bugbear "D", # pydocstyle "EM", # flake8-errmsg "F", # Pyflakes "I", # isort "INP", # flake8-no-pep420 "N", # pep8-naming "PT", # flake8-pytest-style "PTH", # flake8-use-pathlib "RET", # flake8-return "RUF", # Ruff-specific rules "SIM", # flake8-simplify "UP", # pyupgrade ] ignore = [ "D203", # No blank lines between a section header and its content "D205", # 1 blank line required between summary line and description "D212", # Multi-line docstring summary should start at the first line "D400", # First line should end with a period "D401", # First line should be in imperative mood "D415", # First line should end with a period, question mark, or exclamation point "ANN003", # Missing type annotation for kwargs "RUF100", # Unused noqa directive (version-dependent on ruff and not deterministic) ] pydocstyle.convention = "google" [lint.per-file-ignores] "{tests}/*" = [ "D", # pydocstyle "INP", # flake8-no-pep420 "ANN", # flake8-annotations-complexity ] "{devtools}/*" = [ "D", # pydocstyle "INP", # flake8-no-pep420 "F401", # unused import "ANN", # flake8-annotations-complexity ] [format] preview = true docstring-code-format = false docstring-code-line-length = "dynamic" ================================================ FILE: sample/workspace/ABC.Report/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "Report", "displayName": "ABC" }, "config": { "version": "2.0", "logicalId": "40db346f-1c24-8aa6-48c7-a3aec4199351" } } ================================================ FILE: sample/workspace/ABC.Report/StaticResources/SharedResources/BaseThemes/CY24SU10.json ================================================ { "name": "CY24SU10", "dataColors": [ "#118DFF", "#12239E", "#E66C37", "#6B007B", "#E044A7", "#744EC2", "#D9B300", "#D64550", "#197278", "#1AAB40", "#15C6F4", "#4092FF", "#FFA058", "#BE5DC9", "#F472D0", "#B5A1FF", "#C4A200", "#FF8080", "#00DBBC", "#5BD667", "#0091D5", "#4668C5", "#FF6300", "#99008A", "#EC008C", "#533285", "#99700A", "#FF4141", "#1F9A85", "#25891C", "#0057A2", "#002050", "#C94F0F", "#450F54", "#B60064", "#34124F", "#6A5A29", "#1AAB40", "#BA141A", "#0C3D37", "#0B511F" ], "foreground": "#252423", "foregroundNeutralSecondary": "#605E5C", "foregroundNeutralTertiary": "#B3B0AD", "background": "#FFFFFF", "backgroundLight": "#F3F2F1", "backgroundNeutral": "#C8C6C4", "tableAccent": "#118DFF", "good": "#1AAB40", "neutral": "#D9B300", "bad": "#D64554", "maximum": "#118DFF", "center": "#D9B300", "minimum": "#DEEFFF", "null": "#FF7F48", "hyperlink": "#0078d4", "visitedHyperlink": "#0078d4", "textClasses": { "callout": { "fontSize": 45, "fontFace": "DIN", "color": "#252423" }, "title": { "fontSize": 12, "fontFace": "DIN", "color": "#252423" }, "header": { "fontSize": 12, "fontFace": "Segoe UI Semibold", "color": "#252423" }, "label": { "fontSize": 10, "fontFace": "Segoe UI", "color": "#252423" } }, "visualStyles": { "*": { "*": { "*": [ { "wordWrap": true } ], "line": [ { "transparency": 0 } ], "outline": [ { "transparency": 0 } ], "plotArea": [ { "transparency": 0 } ], "categoryAxis": [ { "showAxisTitle": true, "gridlineStyle": "dotted", "concatenateLabels": false } ], "valueAxis": [ { "showAxisTitle": true, "gridlineStyle": "dotted" } ], "y2Axis": [ { "show": true } ], "title": [ { "titleWrap": true } ], "lineStyles": [ { "strokeWidth": 3 } ], "wordWrap": [ { "show": true } ], "background": [ { "show": true, "transparency": 0 } ], "border": [ { "width": 1 } ], "outspacePane": [ { "backgroundColor": { "solid": { "color": "#ffffff" } }, "transparency": 0, "border": true, "borderColor": { "solid": { "color": "#B3B0AD" } } } ], "filterCard": [ { "$id": "Applied", "transparency": 0, "foregroundColor": { "solid": { "color": "#252423" } }, "border": true }, { "$id": "Available", "transparency": 0, "foregroundColor": { "solid": { "color": "#252423" } }, "border": true } ] } }, "scatterChart": { "*": { "bubbles": [ { "bubbleSize": -10, "markerRangeType": "auto" } ], "general": [ { "responsive": true } ], "fillPoint": [ { "show": true } ], "legend": [ { "showGradientLegend": true } ] } }, "lineChart": { "*": { "general": [ { "responsive": true } ], "smallMultiplesLayout": [ { "backgroundTransparency": 0, "gridLineType": "inner" } ], "forecast": [ { "matchSeriesInterpolation": true } ] } }, "map": { "*": { "bubbles": [ { "bubbleSize": -10, "markerRangeType": "auto" } ] } }, "azureMap": { "*": { "bubbleLayer": [ { "bubbleRadius": 8, "minBubbleRadius": 8, "maxRadius": 40 } ], "barChart": [ { "barHeight": 3, "thickness": 3 } ] } }, "pieChart": { "*": { "legend": [ { "show": true, "position": "RightCenter" } ], "labels": [ { "labelStyle": "Data value, percent of total" } ] } }, "donutChart": { "*": { "legend": [ { "show": true, "position": "RightCenter" } ], "labels": [ { "labelStyle": "Data value, percent of total" } ] } }, "pivotTable": { "*": { "rowHeaders": [ { "showExpandCollapseButtons": true, "legacyStyleDisabled": true } ] } }, "multiRowCard": { "*": { "card": [ { "outlineWeight": 2, "barShow": true, "barWeight": 2 } ] } }, "kpi": { "*": { "trendline": [ { "transparency": 20 } ] } }, "cardVisual": { "*": { "layout": [ { "maxTiles": 3 } ], "overflow": [ { "type": 0 } ], "image": [ { "fixedSize": false }, { "imageAreaSize": 50 } ] } }, "advancedSlicerVisual": { "*": { "layout": [ { "maxTiles": 3 } ] } }, "slicer": { "*": { "general": [ { "responsive": true } ], "date": [ { "hideDatePickerButton": false } ], "items": [ { "padding": 4, "accessibilityContrastProperties": true } ] } }, "waterfallChart": { "*": { "general": [ { "responsive": true } ] } }, "columnChart": { "*": { "general": [ { "responsive": true } ], "legend": [ { "showGradientLegend": true } ], "smallMultiplesLayout": [ { "backgroundTransparency": 0, "gridLineType": "inner" } ] } }, "clusteredColumnChart": { "*": { "general": [ { "responsive": true } ], "legend": [ { "showGradientLegend": true } ], "smallMultiplesLayout": [ { "backgroundTransparency": 0, "gridLineType": "inner" } ] } }, "hundredPercentStackedColumnChart": { "*": { "general": [ { "responsive": true } ], "legend": [ { "showGradientLegend": true } ], "smallMultiplesLayout": [ { "backgroundTransparency": 0, "gridLineType": "inner" } ] } }, "barChart": { "*": { "general": [ { "responsive": true } ], "legend": [ { "showGradientLegend": true } ], "smallMultiplesLayout": [ { "backgroundTransparency": 0, "gridLineType": "inner" } ] } }, "clusteredBarChart": { "*": { "general": [ { "responsive": true } ], "legend": [ { "showGradientLegend": true } ], "smallMultiplesLayout": [ { "backgroundTransparency": 0, "gridLineType": "inner" } ] } }, "hundredPercentStackedBarChart": { "*": { "general": [ { "responsive": true } ], "legend": [ { "showGradientLegend": true } ], "smallMultiplesLayout": [ { "backgroundTransparency": 0, "gridLineType": "inner" } ] } }, "areaChart": { "*": { "general": [ { "responsive": true } ], "smallMultiplesLayout": [ { "backgroundTransparency": 0, "gridLineType": "inner" } ] } }, "stackedAreaChart": { "*": { "general": [ { "responsive": true } ], "smallMultiplesLayout": [ { "backgroundTransparency": 0, "gridLineType": "inner" } ] } }, "lineClusteredColumnComboChart": { "*": { "general": [ { "responsive": true } ], "smallMultiplesLayout": [ { "backgroundTransparency": 0, "gridLineType": "inner" } ] } }, "lineStackedColumnComboChart": { "*": { "general": [ { "responsive": true } ], "smallMultiplesLayout": [ { "backgroundTransparency": 0, "gridLineType": "inner" } ] } }, "ribbonChart": { "*": { "general": [ { "responsive": true } ], "smallMultiplesLayout": [ { "backgroundTransparency": 0, "gridLineType": "inner" } ], "valueAxis": [ { "show": true } ] } }, "hundredPercentStackedAreaChart": { "*": { "general": [ { "responsive": true } ], "smallMultiplesLayout": [ { "backgroundTransparency": 0, "gridLineType": "inner" } ] } }, "group": { "*": { "background": [ { "show": false } ] } }, "basicShape": { "*": { "background": [ { "show": false } ], "general": [ { "keepLayerOrder": true } ], "visualHeader": [ { "show": false } ] } }, "shape": { "*": { "background": [ { "show": false } ], "general": [ { "keepLayerOrder": true } ], "visualHeader": [ { "show": false } ] } }, "image": { "*": { "background": [ { "show": false } ], "general": [ { "keepLayerOrder": true } ], "visualHeader": [ { "show": false } ], "lockAspect": [ { "show": true } ] } }, "actionButton": { "*": { "background": [ { "show": false } ], "visualHeader": [ { "show": false } ] } }, "pageNavigator": { "*": { "background": [ { "show": false } ], "visualHeader": [ { "show": false } ] } }, "bookmarkNavigator": { "*": { "background": [ { "show": false } ], "visualHeader": [ { "show": false } ] } }, "textbox": { "*": { "general": [ { "keepLayerOrder": true } ], "visualHeader": [ { "show": false } ] } }, "page": { "*": { "outspace": [ { "color": { "solid": { "color": "#FFFFFF" } } } ], "background": [ { "transparency": 100 } ] } } } } ================================================ FILE: sample/workspace/ABC.Report/definition.pbir ================================================ { "version": "4.0", "datasetReference": { "byPath": { "path": "../ABC.SemanticModel" } } } ================================================ FILE: sample/workspace/ABC.Report/report.json ================================================ { "config": "{\"version\":\"5.59\",\"themeCollection\":{\"baseTheme\":{\"name\":\"CY24SU10\",\"version\":\"5.61\",\"type\":2}},\"activeSectionIndex\":3,\"defaultDrillFilterOtherVisuals\":true,\"linguisticSchemaSyncVersion\":2,\"settings\":{\"useNewFilterPaneExperience\":true,\"allowChangeFilterTypes\":true,\"useStylableVisualContainerHeader\":true,\"queryLimitOption\":6,\"useEnhancedTooltips\":true,\"exportDataMode\":1,\"useDefaultAggregateDisplayName\":true},\"objects\":{\"section\":[{\"properties\":{\"verticalAlignment\":{\"expr\":{\"Literal\":{\"Value\":\"'Top'\"}}}}}]}}", "layoutOptimization": 0, "resourcePackages": [ { "resourcePackage": { "disabled": false, "items": [ { "name": "CY24SU10", "path": "BaseThemes/CY24SU10.json", "type": 202 } ], "name": "SharedResources", "type": 2 } }, { "resourcePackage": { "disabled": false, "items": [ { "name": "test_image13726994662591707.bmp", "path": "test_image13726994662591707.bmp", "type": 100 }, { "name": "test_image15532910574958114.gif", "path": "test_image15532910574958114.gif", "type": 100 }, { "name": "test_image25861853181026917.tif", "path": "test_image25861853181026917.tif", "type": 100 }, { "name": "test_image9544675947263683.jpg", "path": "test_image9544675947263683.jpg", "type": 100 }, { "name": "test_image9896482226816141.png", "path": "test_image9896482226816141.png", "type": 100 } ], "name": "RegisteredResources", "type": 1 } } ], "sections": [ { "config": "{}", "displayName": "Page 3", "displayOption": 1, "filters": "[]", "height": 720.00, "name": "63e290aa19e7bea3d03d", "ordinal": 2, "visualContainers": [ { "config": "{\"name\":\"39eb354ddb07467c1948\",\"layouts\":[{\"id\":0,\"position\":{\"x\":525.312,\"y\":169.984,\"z\":0,\"width\":279.552,\"height\":279.552,\"tabOrder\":0}}],\"singleVisual\":{\"visualType\":\"hundredPercentStackedBarChart\",\"projections\":{\"Category\":[{\"queryRef\":\"Table.Column1\",\"active\":true}]},\"prototypeQuery\":{\"Version\":2,\"From\":[{\"Name\":\"t\",\"Entity\":\"Table\",\"Type\":0}],\"Select\":[{\"Column\":{\"Expression\":{\"SourceRef\":{\"Source\":\"t\"}},\"Property\":\"Column1\"},\"Name\":\"Table.Column1\",\"NativeReferenceName\":\"Column1\"}]},\"drillFilterOtherVisuals\":true}}", "filters": "[]", "height": 279.55, "width": 279.55, "x": 525.31, "y": 169.98, "z": 0.00 } ], "width": 1280.00 }, { "config": "{}", "displayName": "Page 1", "displayOption": 1, "filters": "[]", "height": 720.00, "name": "90cb47b35aee21c5a77c", "visualContainers": [ { "config": "{\"name\":\"d15aa7ce4fc6c2acb4f2\",\"layouts\":[{\"id\":0,\"position\":{\"x\":342,\"y\":104,\"z\":0,\"width\":280,\"height\":280,\"tabOrder\":0}}],\"singleVisual\":{\"visualType\":\"tableEx\",\"projections\":{\"Values\":[{\"queryRef\":\"Table.Column1\"},{\"queryRef\":\"Table.Column2\"}]},\"prototypeQuery\":{\"Version\":2,\"From\":[{\"Name\":\"t\",\"Entity\":\"Table\",\"Type\":0}],\"Select\":[{\"Column\":{\"Expression\":{\"SourceRef\":{\"Source\":\"t\"}},\"Property\":\"Column1\"},\"Name\":\"Table.Column1\",\"NativeReferenceName\":\"Column1\"},{\"Column\":{\"Expression\":{\"SourceRef\":{\"Source\":\"t\"}},\"Property\":\"Column2\"},\"Name\":\"Table.Column2\",\"NativeReferenceName\":\"Column2\"}]},\"drillFilterOtherVisuals\":true}}", "filters": "[]", "height": 280.00, "width": 280.00, "x": 342.00, "y": 104.00, "z": 0.00 } ], "width": 1280.00 }, { "config": "{}", "displayName": "Page 4", "displayOption": 1, "filters": "[]", "height": 720.00, "name": "c744048b570d475206d0", "ordinal": 3, "visualContainers": [ { "config": "{\"name\":\"63c15486d69a1d77ad82\",\"layouts\":[{\"id\":0,\"position\":{\"x\":937.5,\"y\":27,\"width\":161,\"height\":161,\"z\":4,\"tabOrder\":4}}],\"singleVisual\":{\"visualType\":\"image\",\"drillFilterOtherVisuals\":true,\"objects\":{\"general\":[{\"properties\":{\"imageUrl\":{\"expr\":{\"ResourcePackageItem\":{\"PackageName\":\"RegisteredResources\",\"PackageType\":1,\"ItemName\":\"test_image25861853181026917.tif\"}}}}}]}},\"howCreated\":\"InsertVisualButton\"}", "filters": "[]", "height": 161.00, "width": 161.00, "x": 937.50, "y": 27.00, "z": 4.00 }, { "config": "{\"name\":\"6ac48140b12eb0ab5bac\",\"layouts\":[{\"id\":0,\"position\":{\"x\":22.426995457495135,\"y\":49.00713822193381,\"z\":0,\"width\":161.14211550940948,\"height\":161.14211550940948,\"tabOrder\":0}}],\"singleVisual\":{\"visualType\":\"image\",\"drillFilterOtherVisuals\":true,\"objects\":{\"general\":[{\"properties\":{\"imageUrl\":{\"expr\":{\"ResourcePackageItem\":{\"PackageName\":\"RegisteredResources\",\"PackageType\":1,\"ItemName\":\"test_image13726994662591707.bmp\"}}}}}]}},\"howCreated\":\"InsertVisualButton\"}", "filters": "[]", "height": 161.14, "width": 161.14, "x": 22.43, "y": 49.01, "z": 0.00 }, { "config": "{\"name\":\"8bce33fe60a0c78d1459\",\"layouts\":[{\"id\":0,\"position\":{\"x\":224.26995457495133,\"y\":49.00713822193381,\"z\":1,\"width\":161.14211550940948,\"height\":161.14211550940948,\"tabOrder\":1}}],\"singleVisual\":{\"visualType\":\"image\",\"drillFilterOtherVisuals\":true,\"objects\":{\"general\":[{\"properties\":{\"imageUrl\":{\"expr\":{\"ResourcePackageItem\":{\"PackageName\":\"RegisteredResources\",\"PackageType\":1,\"ItemName\":\"test_image15532910574958114.gif\"}}}}}]}},\"howCreated\":\"InsertVisualButton\"}", "filters": "[]", "height": 161.14, "width": 161.14, "x": 224.27, "y": 49.01, "z": 1.00 }, { "config": "{\"name\":\"ac2f524856312c1c0882\",\"layouts\":[{\"id\":0,\"position\":{\"x\":706.8656716417911,\"y\":27.410772225827387,\"z\":3,\"width\":161.14211550940948,\"height\":161.14211550940948,\"tabOrder\":3}}],\"singleVisual\":{\"visualType\":\"image\",\"drillFilterOtherVisuals\":true,\"objects\":{\"general\":[{\"properties\":{\"imageUrl\":{\"expr\":{\"ResourcePackageItem\":{\"PackageName\":\"RegisteredResources\",\"PackageType\":1,\"ItemName\":\"test_image9896482226816141.png\"}}}}}]}},\"howCreated\":\"InsertVisualButton\"}", "filters": "[]", "height": 161.14, "width": 161.14, "x": 706.87, "y": 27.41, "z": 3.00 }, { "config": "{\"name\":\"bb2e1aaeee837cbc07c6\",\"layouts\":[{\"id\":0,\"position\":{\"x\":443.5561323815704,\"y\":49.00713822193381,\"z\":2,\"width\":161.14211550940948,\"height\":161.14211550940948,\"tabOrder\":2}}],\"singleVisual\":{\"visualType\":\"image\",\"drillFilterOtherVisuals\":true,\"objects\":{\"general\":[{\"properties\":{\"imageUrl\":{\"expr\":{\"ResourcePackageItem\":{\"PackageName\":\"RegisteredResources\",\"PackageType\":1,\"ItemName\":\"test_image9544675947263683.jpg\"}}}}}]}},\"howCreated\":\"InsertVisualButton\"}", "filters": "[]", "height": 161.14, "width": 161.14, "x": 443.56, "y": 49.01, "z": 2.00 } ], "width": 1280.00 }, { "config": "{}", "displayName": "Page 2", "displayOption": 1, "filters": "[]", "height": 720.00, "name": "f072cad6a715c38915c7", "ordinal": 1, "visualContainers": [ { "config": "{\"name\":\"1e78447fe04d60e1956f\",\"layouts\":[{\"id\":0,\"position\":{\"x\":317.44,\"y\":69.632,\"z\":0,\"width\":279.552,\"height\":279.552,\"tabOrder\":0}}],\"singleVisual\":{\"visualType\":\"areaChart\",\"projections\":{\"Category\":[{\"queryRef\":\"Table.Column1\",\"active\":true}],\"Y\":[{\"queryRef\":\"CountNonNull(Table.Column2)\"}],\"Series\":[{\"queryRef\":\"Table_2.Column1\"}]},\"prototypeQuery\":{\"Version\":2,\"From\":[{\"Name\":\"t\",\"Entity\":\"Table\",\"Type\":0},{\"Name\":\"t1\",\"Entity\":\"Table_2\",\"Type\":0}],\"Select\":[{\"Column\":{\"Expression\":{\"SourceRef\":{\"Source\":\"t\"}},\"Property\":\"Column1\"},\"Name\":\"Table.Column1\",\"NativeReferenceName\":\"Column1\"},{\"Aggregation\":{\"Expression\":{\"Column\":{\"Expression\":{\"SourceRef\":{\"Source\":\"t\"}},\"Property\":\"Column2\"}},\"Function\":5},\"Name\":\"CountNonNull(Table.Column2)\",\"NativeReferenceName\":\"Column2\"},{\"Column\":{\"Expression\":{\"SourceRef\":{\"Source\":\"t1\"}},\"Property\":\"Column1\"},\"Name\":\"Table_2.Column1\",\"NativeReferenceName\":\"Column11\"}],\"OrderBy\":[{\"Direction\":2,\"Expression\":{\"Aggregation\":{\"Expression\":{\"Column\":{\"Expression\":{\"SourceRef\":{\"Source\":\"t\"}},\"Property\":\"Column2\"}},\"Function\":5}}}]},\"drillFilterOtherVisuals\":true,\"hasDefaultSort\":true}}", "filters": "[]", "height": 279.55, "width": 279.55, "x": 317.44, "y": 69.63, "z": 0.00 } ], "width": 1280.00 } ] } ================================================ FILE: sample/workspace/ABC.SemanticModel/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "SemanticModel", "displayName": "ABC" }, "config": { "version": "2.0", "logicalId": "5ead2a08-6831-80df-4110-6265e29e3ee7" } } ================================================ FILE: sample/workspace/ABC.SemanticModel/definition/cultures/en-US.tmdl ================================================ cultureInfo en-US linguisticMetadata = { "Version": "1.0.0", "Language": "en-US" } contentType: json ================================================ FILE: sample/workspace/ABC.SemanticModel/definition/database.tmdl ================================================ database compatibilityLevel: 1550 ================================================ FILE: sample/workspace/ABC.SemanticModel/definition/model.tmdl ================================================ model Model culture: en-US defaultPowerBIDataSourceVersion: powerBI_V3 sourceQueryCulture: en-US dataAccessOptions legacyRedirects returnErrorValuesAsNull annotation PBI_QueryOrder = ["Table","Table_2"] annotation __PBI_TimeIntelligenceEnabled = 1 annotation PBIDesktopVersion = 2.140.679.0 (25.02)+e2f7796f7ddf473b3f87e4d9e2bee0b29f9956bf annotation PBI_ProTooling = ["DevMode"] ref table Table ref table Table_2 ref cultureInfo en-US ================================================ FILE: sample/workspace/ABC.SemanticModel/definition/relationships.tmdl ================================================ relationship AutoDetected_981379f6-d1c0-48e8-a4d4-cf3718d1fc57 crossFilteringBehavior: bothDirections fromCardinality: one fromColumn: Table.Column2 toColumn: Table_2.Column1 ================================================ FILE: sample/workspace/ABC.SemanticModel/definition/tables/Table.tmdl ================================================ table Table lineageTag: e9a135a4-d745-4125-9c08-01f5a8ee959e column Column1 dataType: string lineageTag: 2b0b4597-fb3d-432e-9047-671b8e2a3970 summarizeBy: none sourceColumn: Column1 annotation SummarizationSetBy = Automatic column Column2 dataType: int64 formatString: 0 lineageTag: def47816-d442-4e60-aed3-6b05c2fb0265 summarizeBy: none sourceColumn: Column2 annotation SummarizationSetBy = Automatic partition Table = m mode: import source = let Source = Table.FromRows(Json.Document(Binary.Decompress(Binary.FromText("i45WclTSUTJUitWJVnICsozALGcgy1gpNhYA", BinaryEncoding.Base64), Compression.Deflate)), let _t = ((type nullable text) meta [Serialized.Text = true]) in type table [Column1 = _t, Column2 = _t]), #"Changed Type" = Table.TransformColumnTypes(Source,{{"Column1", type text}, {"Column2", Int64.Type}}) in #"Changed Type" annotation PBI_NavigationStepName = Navigation annotation PBI_ResultType = Table ================================================ FILE: sample/workspace/ABC.SemanticModel/definition/tables/Table_2.tmdl ================================================ table Table_2 lineageTag: 2f801d69-7502-4625-aba2-356b8f4378b2 column Column1 dataType: int64 formatString: 0 lineageTag: adbc29fb-d481-4ebf-96b8-ed12b1d1d7e1 summarizeBy: none sourceColumn: Column1 annotation SummarizationSetBy = Automatic partition Table_2 = m mode: import source = let Source = Table.FromRows(Json.Document(Binary.Decompress(Binary.FromText("i45WMlSK1YlWMgKTxmDSBEyaKsXGAgA=", BinaryEncoding.Base64), Compression.Deflate)), let _t = ((type nullable text) meta [Serialized.Text = true]) in type table [Column1 = _t]), #"Changed Type" = Table.TransformColumnTypes(Source,{{"Column1", Int64.Type}}) in #"Changed Type" annotation PBI_NavigationStepName = Navigation annotation PBI_ResultType = Table ================================================ FILE: sample/workspace/ABC.SemanticModel/definition.pbism ================================================ { "version": "4.0", "settings": {} } ================================================ FILE: sample/workspace/ABC.SemanticModel/diagramLayout.json ================================================ { "version": "1.1.0", "diagrams": [ { "ordinal": 0, "scrollPosition": { "x": 0, "y": 0 }, "nodes": [ { "location": { "x": 294, "y": -10 }, "nodeIndex": "Table", "nodeLineageTag": "e9a135a4-d745-4125-9c08-01f5a8ee959e", "size": { "height": 128, "width": 234 }, "zIndex": 0 }, { "location": { "x": 10, "y": 2 }, "nodeIndex": "Table_2", "nodeLineageTag": "2f801d69-7502-4625-aba2-356b8f4378b2", "size": { "height": 104, "width": 234 }, "zIndex": 0 } ], "name": "All tables", "zoomValue": 100, "pinKeyFieldsToTop": false, "showExtraHeaderInfo": false, "hideKeyFieldsWhenCollapsed": false, "tablesLocked": false } ], "selectedDiagram": "All tables", "defaultDiagram": "All tables" } ================================================ FILE: sample/workspace/ABCD.Report/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "Report", "displayName": "ABCD" }, "config": { "version": "2.0", "logicalId": "40db346f-1c24-8aa6-48c7-a3aec4199350" } } ================================================ FILE: sample/workspace/ABCD.Report/StaticResources/SharedResources/BaseThemes/CY24SU10.json ================================================ { "name": "CY24SU10", "dataColors": [ "#118DFF", "#12239E", "#E66C37", "#6B007B", "#E044A7", "#744EC2", "#D9B300", "#D64550", "#197278", "#1AAB40", "#15C6F4", "#4092FF", "#FFA058", "#BE5DC9", "#F472D0", "#B5A1FF", "#C4A200", "#FF8080", "#00DBBC", "#5BD667", "#0091D5", "#4668C5", "#FF6300", "#99008A", "#EC008C", "#533285", "#99700A", "#FF4141", "#1F9A85", "#25891C", "#0057A2", "#002050", "#C94F0F", "#450F54", "#B60064", "#34124F", "#6A5A29", "#1AAB40", "#BA141A", "#0C3D37", "#0B511F" ], "foreground": "#252423", "foregroundNeutralSecondary": "#605E5C", "foregroundNeutralTertiary": "#B3B0AD", "background": "#FFFFFF", "backgroundLight": "#F3F2F1", "backgroundNeutral": "#C8C6C4", "tableAccent": "#118DFF", "good": "#1AAB40", "neutral": "#D9B300", "bad": "#D64554", "maximum": "#118DFF", "center": "#D9B300", "minimum": "#DEEFFF", "null": "#FF7F48", "hyperlink": "#0078d4", "visitedHyperlink": "#0078d4", "textClasses": { "callout": { "fontSize": 45, "fontFace": "DIN", "color": "#252423" }, "title": { "fontSize": 12, "fontFace": "DIN", "color": "#252423" }, "header": { "fontSize": 12, "fontFace": "Segoe UI Semibold", "color": "#252423" }, "label": { "fontSize": 10, "fontFace": "Segoe UI", "color": "#252423" } }, "visualStyles": { "*": { "*": { "*": [ { "wordWrap": true } ], "line": [ { "transparency": 0 } ], "outline": [ { "transparency": 0 } ], "plotArea": [ { "transparency": 0 } ], "categoryAxis": [ { "showAxisTitle": true, "gridlineStyle": "dotted", "concatenateLabels": false } ], "valueAxis": [ { "showAxisTitle": true, "gridlineStyle": "dotted" } ], "y2Axis": [ { "show": true } ], "title": [ { "titleWrap": true } ], "lineStyles": [ { "strokeWidth": 3 } ], "wordWrap": [ { "show": true } ], "background": [ { "show": true, "transparency": 0 } ], "border": [ { "width": 1 } ], "outspacePane": [ { "backgroundColor": { "solid": { "color": "#ffffff" } }, "transparency": 0, "border": true, "borderColor": { "solid": { "color": "#B3B0AD" } } } ], "filterCard": [ { "$id": "Applied", "transparency": 0, "foregroundColor": { "solid": { "color": "#252423" } }, "border": true }, { "$id": "Available", "transparency": 0, "foregroundColor": { "solid": { "color": "#252423" } }, "border": true } ] } }, "scatterChart": { "*": { "bubbles": [ { "bubbleSize": -10, "markerRangeType": "auto" } ], "general": [ { "responsive": true } ], "fillPoint": [ { "show": true } ], "legend": [ { "showGradientLegend": true } ] } }, "lineChart": { "*": { "general": [ { "responsive": true } ], "smallMultiplesLayout": [ { "backgroundTransparency": 0, "gridLineType": "inner" } ], "forecast": [ { "matchSeriesInterpolation": true } ] } }, "map": { "*": { "bubbles": [ { "bubbleSize": -10, "markerRangeType": "auto" } ] } }, "azureMap": { "*": { "bubbleLayer": [ { "bubbleRadius": 8, "minBubbleRadius": 8, "maxRadius": 40 } ], "barChart": [ { "barHeight": 3, "thickness": 3 } ] } }, "pieChart": { "*": { "legend": [ { "show": true, "position": "RightCenter" } ], "labels": [ { "labelStyle": "Data value, percent of total" } ] } }, "donutChart": { "*": { "legend": [ { "show": true, "position": "RightCenter" } ], "labels": [ { "labelStyle": "Data value, percent of total" } ] } }, "pivotTable": { "*": { "rowHeaders": [ { "showExpandCollapseButtons": true, "legacyStyleDisabled": true } ] } }, "multiRowCard": { "*": { "card": [ { "outlineWeight": 2, "barShow": true, "barWeight": 2 } ] } }, "kpi": { "*": { "trendline": [ { "transparency": 20 } ] } }, "cardVisual": { "*": { "layout": [ { "maxTiles": 3 } ], "overflow": [ { "type": 0 } ], "image": [ { "fixedSize": false }, { "imageAreaSize": 50 } ] } }, "advancedSlicerVisual": { "*": { "layout": [ { "maxTiles": 3 } ] } }, "slicer": { "*": { "general": [ { "responsive": true } ], "date": [ { "hideDatePickerButton": false } ], "items": [ { "padding": 4, "accessibilityContrastProperties": true } ] } }, "waterfallChart": { "*": { "general": [ { "responsive": true } ] } }, "columnChart": { "*": { "general": [ { "responsive": true } ], "legend": [ { "showGradientLegend": true } ], "smallMultiplesLayout": [ { "backgroundTransparency": 0, "gridLineType": "inner" } ] } }, "clusteredColumnChart": { "*": { "general": [ { "responsive": true } ], "legend": [ { "showGradientLegend": true } ], "smallMultiplesLayout": [ { "backgroundTransparency": 0, "gridLineType": "inner" } ] } }, "hundredPercentStackedColumnChart": { "*": { "general": [ { "responsive": true } ], "legend": [ { "showGradientLegend": true } ], "smallMultiplesLayout": [ { "backgroundTransparency": 0, "gridLineType": "inner" } ] } }, "barChart": { "*": { "general": [ { "responsive": true } ], "legend": [ { "showGradientLegend": true } ], "smallMultiplesLayout": [ { "backgroundTransparency": 0, "gridLineType": "inner" } ] } }, "clusteredBarChart": { "*": { "general": [ { "responsive": true } ], "legend": [ { "showGradientLegend": true } ], "smallMultiplesLayout": [ { "backgroundTransparency": 0, "gridLineType": "inner" } ] } }, "hundredPercentStackedBarChart": { "*": { "general": [ { "responsive": true } ], "legend": [ { "showGradientLegend": true } ], "smallMultiplesLayout": [ { "backgroundTransparency": 0, "gridLineType": "inner" } ] } }, "areaChart": { "*": { "general": [ { "responsive": true } ], "smallMultiplesLayout": [ { "backgroundTransparency": 0, "gridLineType": "inner" } ] } }, "stackedAreaChart": { "*": { "general": [ { "responsive": true } ], "smallMultiplesLayout": [ { "backgroundTransparency": 0, "gridLineType": "inner" } ] } }, "lineClusteredColumnComboChart": { "*": { "general": [ { "responsive": true } ], "smallMultiplesLayout": [ { "backgroundTransparency": 0, "gridLineType": "inner" } ] } }, "lineStackedColumnComboChart": { "*": { "general": [ { "responsive": true } ], "smallMultiplesLayout": [ { "backgroundTransparency": 0, "gridLineType": "inner" } ] } }, "ribbonChart": { "*": { "general": [ { "responsive": true } ], "smallMultiplesLayout": [ { "backgroundTransparency": 0, "gridLineType": "inner" } ], "valueAxis": [ { "show": true } ] } }, "hundredPercentStackedAreaChart": { "*": { "general": [ { "responsive": true } ], "smallMultiplesLayout": [ { "backgroundTransparency": 0, "gridLineType": "inner" } ] } }, "group": { "*": { "background": [ { "show": false } ] } }, "basicShape": { "*": { "background": [ { "show": false } ], "general": [ { "keepLayerOrder": true } ], "visualHeader": [ { "show": false } ] } }, "shape": { "*": { "background": [ { "show": false } ], "general": [ { "keepLayerOrder": true } ], "visualHeader": [ { "show": false } ] } }, "image": { "*": { "background": [ { "show": false } ], "general": [ { "keepLayerOrder": true } ], "visualHeader": [ { "show": false } ], "lockAspect": [ { "show": true } ] } }, "actionButton": { "*": { "background": [ { "show": false } ], "visualHeader": [ { "show": false } ] } }, "pageNavigator": { "*": { "background": [ { "show": false } ], "visualHeader": [ { "show": false } ] } }, "bookmarkNavigator": { "*": { "background": [ { "show": false } ], "visualHeader": [ { "show": false } ] } }, "textbox": { "*": { "general": [ { "keepLayerOrder": true } ], "visualHeader": [ { "show": false } ] } }, "page": { "*": { "outspace": [ { "color": { "solid": { "color": "#FFFFFF" } } } ], "background": [ { "transparency": 100 } ] } } } } ================================================ FILE: sample/workspace/ABCD.Report/definition.pbir ================================================ { "version": "4.0", "datasetReference": { "byPath": { "path": "../ABC.SemanticModel" } } } ================================================ FILE: sample/workspace/ABCD.Report/report.json ================================================ { "config": "{\"version\":\"5.59\",\"themeCollection\":{\"baseTheme\":{\"name\":\"CY24SU10\",\"version\":\"5.61\",\"type\":2}},\"activeSectionIndex\":0,\"defaultDrillFilterOtherVisuals\":true,\"linguisticSchemaSyncVersion\":2,\"settings\":{\"useNewFilterPaneExperience\":true,\"allowChangeFilterTypes\":true,\"useStylableVisualContainerHeader\":true,\"queryLimitOption\":6,\"useEnhancedTooltips\":true,\"exportDataMode\":1,\"useDefaultAggregateDisplayName\":true},\"objects\":{\"section\":[{\"properties\":{\"verticalAlignment\":{\"expr\":{\"Literal\":{\"Value\":\"'Top'\"}}}}}]}}", "layoutOptimization": 0, "resourcePackages": [ { "resourcePackage": { "disabled": false, "items": [ { "name": "CY24SU10", "path": "BaseThemes/CY24SU10.json", "type": 202 } ], "name": "SharedResources", "type": 2 } } ], "sections": [ { "config": "{}", "displayName": "Page 3", "displayOption": 1, "filters": "[]", "height": 720.00, "name": "63e290aa19e7bea3d03d", "ordinal": 2, "visualContainers": [ { "config": "{\"name\":\"39eb354ddb07467c1948\",\"layouts\":[{\"id\":0,\"position\":{\"x\":525.312,\"y\":169.984,\"z\":0,\"width\":279.552,\"height\":279.552,\"tabOrder\":0}}],\"singleVisual\":{\"visualType\":\"hundredPercentStackedBarChart\",\"projections\":{\"Category\":[{\"queryRef\":\"Table.Column1\",\"active\":true}]},\"prototypeQuery\":{\"Version\":2,\"From\":[{\"Name\":\"t\",\"Entity\":\"Table\",\"Type\":0}],\"Select\":[{\"Column\":{\"Expression\":{\"SourceRef\":{\"Source\":\"t\"}},\"Property\":\"Column1\"},\"Name\":\"Table.Column1\",\"NativeReferenceName\":\"Column1\"}]},\"drillFilterOtherVisuals\":true}}", "filters": "[]", "height": 279.55, "width": 279.55, "x": 525.31, "y": 169.98, "z": 0.00 } ], "width": 1280.00 }, { "config": "{}", "displayName": "Page 1", "displayOption": 1, "filters": "[]", "height": 720.00, "name": "90cb47b35aee21c5a77c", "visualContainers": [ { "config": "{\"name\":\"d15aa7ce4fc6c2acb4f2\",\"layouts\":[{\"id\":0,\"position\":{\"x\":342,\"y\":104,\"z\":0,\"width\":280,\"height\":280,\"tabOrder\":0}}],\"singleVisual\":{\"visualType\":\"tableEx\",\"projections\":{\"Values\":[{\"queryRef\":\"Table.Column1\"},{\"queryRef\":\"Table.Column2\"}]},\"prototypeQuery\":{\"Version\":2,\"From\":[{\"Name\":\"t\",\"Entity\":\"Table\",\"Type\":0}],\"Select\":[{\"Column\":{\"Expression\":{\"SourceRef\":{\"Source\":\"t\"}},\"Property\":\"Column1\"},\"Name\":\"Table.Column1\",\"NativeReferenceName\":\"Column1\"},{\"Column\":{\"Expression\":{\"SourceRef\":{\"Source\":\"t\"}},\"Property\":\"Column2\"},\"Name\":\"Table.Column2\",\"NativeReferenceName\":\"Column2\"}]},\"drillFilterOtherVisuals\":true}}", "filters": "[]", "height": 280.00, "width": 280.00, "x": 342.00, "y": 104.00, "z": 0.00 } ], "width": 1280.00 }, { "config": "{}", "displayName": "Page 2", "displayOption": 1, "filters": "[]", "height": 720.00, "name": "f072cad6a715c38915c7", "ordinal": 1, "visualContainers": [ { "config": "{\"name\":\"1e78447fe04d60e1956f\",\"layouts\":[{\"id\":0,\"position\":{\"x\":317.44,\"y\":69.632,\"z\":0,\"width\":279.552,\"height\":279.552,\"tabOrder\":0}}],\"singleVisual\":{\"visualType\":\"areaChart\",\"projections\":{\"Category\":[{\"queryRef\":\"Table.Column1\",\"active\":true}],\"Y\":[{\"queryRef\":\"CountNonNull(Table.Column2)\"}],\"Series\":[{\"queryRef\":\"Table_2.Column1\"}]},\"prototypeQuery\":{\"Version\":2,\"From\":[{\"Name\":\"t\",\"Entity\":\"Table\",\"Type\":0},{\"Name\":\"t1\",\"Entity\":\"Table_2\",\"Type\":0}],\"Select\":[{\"Column\":{\"Expression\":{\"SourceRef\":{\"Source\":\"t\"}},\"Property\":\"Column1\"},\"Name\":\"Table.Column1\",\"NativeReferenceName\":\"Column1\"},{\"Aggregation\":{\"Expression\":{\"Column\":{\"Expression\":{\"SourceRef\":{\"Source\":\"t\"}},\"Property\":\"Column2\"}},\"Function\":5},\"Name\":\"CountNonNull(Table.Column2)\",\"NativeReferenceName\":\"Column2\"},{\"Column\":{\"Expression\":{\"SourceRef\":{\"Source\":\"t1\"}},\"Property\":\"Column1\"},\"Name\":\"Table_2.Column1\",\"NativeReferenceName\":\"Column11\"}],\"OrderBy\":[{\"Direction\":2,\"Expression\":{\"Aggregation\":{\"Expression\":{\"Column\":{\"Expression\":{\"SourceRef\":{\"Source\":\"t\"}},\"Property\":\"Column2\"}},\"Function\":5}}}]},\"drillFilterOtherVisuals\":true,\"hasDefaultSort\":true}}", "filters": "[]", "height": 279.55, "width": 279.55, "x": 317.44, "y": 69.63, "z": 0.00 } ], "width": 1280.00 } ] } ================================================ FILE: sample/workspace/ByConnection.Report/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "Report", "displayName": "ByConnection" }, "config": { "version": "2.0", "logicalId": "12345678-1234-1234-1234-123456789abc" } } ================================================ FILE: sample/workspace/ByConnection.Report/definition.pbir ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/report/definitionProperties/2.0.0/schema.json", "version": "4.0", "datasetReference": { "byConnection": { "connectionString": "Data Source=powerbi://api.powerbi.com/v1.0/myorg/dev-workspace-id;initial catalog=dev-semantic-model;access mode=readonly;integrated security=ClaimsToken;semanticmodelid=00000000-0000-0000-0000-000000000000" } } } ================================================ FILE: sample/workspace/ByConnection.Report/report.json ================================================ { "config": "{\"version\":\"5.59\",\"themeCollection\":{\"baseTheme\":{\"name\":\"CY24SU10\",\"version\":\"5.61\",\"type\":2}},\"activeSectionIndex\":0,\"defaultDrillFilterOtherVisuals\":true,\"linguisticSchemaSyncVersion\":2,\"settings\":{\"useNewFilterPaneExperience\":true,\"allowChangeFilterTypes\":true,\"useStylableVisualContainerHeader\":true,\"queryLimitOption\":6,\"useEnhancedTooltips\":true,\"exportDataMode\":1,\"useDefaultAggregateDisplayName\":true}}", "layoutOptimization": 0, "resourcePackages": [ { "resourcePackage": { "disabled": false, "items": [ { "name": "CY24SU10", "path": "BaseThemes/CY24SU10.json", "type": 202 } ], "name": "SharedResources", "type": 2 } } ], "sections": [ { "config": "{}", "displayName": "Page 1", "displayOption": 1, "filters": "[]", "height": 720.00, "name": "ReportSection", "visualContainers": [], "width": 1280.00 } ] } ================================================ FILE: sample/workspace/Default.Warehouse/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "Warehouse", "displayName": "Default" }, "config": { "version": "2.0", "logicalId": "4f344f2d-b51a-a14e-4706-c94957a91175" } } ================================================ FILE: sample/workspace/DefaultCaseInsensitive.Warehouse/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "Warehouse", "displayName": "DefaultCaseInsensitive", "creationPayload": { "defaultCollation": "Latin1_General_100_CI_AS_KS_WS_SC_UTF8" } }, "config": { "version": "2.0", "logicalId": "0e63f8ad-3b03-4f91-8d5e-b33750b3beec" } } ================================================ FILE: sample/workspace/Example Notebook.Notebook/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "Notebook", "displayName": "Example Notebook", "description": "New notebook" }, "config": { "version": "2.0", "logicalId": "21d371bb-4ca3-bf25-46cf-c634f603cbf1" } } ================================================ FILE: sample/workspace/Example Notebook.Notebook/notebook-content.py ================================================ # Fabric notebook source # METADATA ******************** # META { # META "kernel_info": { # META "name": "synapse_pyspark" # META }, # META "dependencies": { # META "lakehouse": { # META "default_lakehouse": "f3b9c1e2-7d4a-4b3e-9f2a-1c6e8d9a7b3c", # META "default_lakehouse_name": "WithoutSchema", # META "default_lakehouse_workspace_id": "c7e4b9f1-2a3d-4e6f-9c8b-7d2f1a4e5b6d", # META "known_lakehouses": [ # META { # META "id": "f3b9c1e2-7d4a-4b3e-9f2a-1c6e8d9a7b3c" # META } # META ] # META } # META } # META } # CELL ******************** # Welcome to your new notebook # Type here in the cell editor to add code! print("This notebook is attached to the 'WithoutSchema' lakehouse.") print("This its SQL analytics endpoint: sqlserverconnectionstringinoriginlakehouse.com") print("This workspace includes 'SampleEventhouse' eventhouse.") print("This its eventhouse query URI: https://trd-origineventhouse.z4.kusto.fabric.microsoft.com") # METADATA ******************** # META { # META "language": "python", # META "language_group": "synapse_pyspark" # META } ================================================ FILE: sample/workspace/Hello Copy Job.CopyJob/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "CopyJob", "displayName": "Hello Copy Job" }, "config": { "version": "2.0", "logicalId": "b0eb37b3-73d6-bae9-40a6-d47ae162a23b" } } ================================================ FILE: sample/workspace/Hello Copy Job.CopyJob/copyjob-content.json ================================================ { "properties": { "jobMode": "Batch", "source": { "type": "LakehouseTable", "connectionSettings": { "type": "Lakehouse", "typeProperties": { "workspaceId": "e96609ad-cc50-4c63-8829-c8499910e044", "artifactId": "0d88b8d7-e73a-418c-8b6c-2a4016602f45", "rootFolder": "Tables" } } }, "destination": { "type": "LakehouseTable", "connectionSettings": { "type": "Lakehouse", "typeProperties": { "workspaceId": "e96609ad-cc50-4c63-8829-c8499910e044", "artifactId": "d0e31750-29de-4992-b01d-ed022494141f", "rootFolder": "Tables" } } }, "policy": { "timeout": "0.12:00:00" } }, "activities": [ { "id": "2905df0f-1421-42e5-b769-0b0a32cb5321", "properties": { "source": { "datasetSettings": { "schema": "dbo", "table": "dimension_city" } }, "destination": { "partitionOption": "None", "writeBehavior": "Overwrite", "datasetSettings": { "schema": "dbo", "table": "dimension_city" } }, "enableStaging": false, "translator": { "type": "TabularTranslator" }, "typeConversionSettings": { "typeConversion": { "allowDataTruncation": true, "treatBooleanAsNumber": false } } } }, { "id": "468840f4-e88a-42a4-b96c-4b95d0c3cd3a", "properties": { "source": { "datasetSettings": { "schema": "dbo", "table": "dimension_customer" } }, "destination": { "partitionOption": "None", "writeBehavior": "Overwrite", "datasetSettings": { "schema": "dbo", "table": "dimension_customer" } }, "enableStaging": false, "translator": { "type": "TabularTranslator" }, "typeConversionSettings": { "typeConversion": { "allowDataTruncation": true, "treatBooleanAsNumber": false } } } }, { "id": "27c592c0-21d1-4d19-b193-0eac8a2b864d", "properties": { "source": { "datasetSettings": { "schema": "dbo", "table": "dimension_date" } }, "destination": { "partitionOption": "None", "writeBehavior": "Overwrite", "datasetSettings": { "schema": "dbo", "table": "dimension_date" } }, "enableStaging": false, "translator": { "type": "TabularTranslator" }, "typeConversionSettings": { "typeConversion": { "allowDataTruncation": true, "treatBooleanAsNumber": false } } } }, { "id": "a9b14f12-b243-45f9-ab21-7642ae6c3afa", "properties": { "source": { "datasetSettings": { "schema": "dbo", "table": "dimension_employee" } }, "destination": { "partitionOption": "None", "writeBehavior": "Overwrite", "datasetSettings": { "schema": "dbo", "table": "dimension_employee" } }, "enableStaging": false, "translator": { "type": "TabularTranslator" }, "typeConversionSettings": { "typeConversion": { "allowDataTruncation": true, "treatBooleanAsNumber": false } } } }, { "id": "f76f4474-489b-4ceb-b885-a2a33df301c9", "properties": { "source": { "datasetSettings": { "schema": "dbo", "table": "dimension_stock_item" } }, "destination": { "partitionOption": "None", "writeBehavior": "Overwrite", "datasetSettings": { "schema": "dbo", "table": "dimension_stock_item" } }, "enableStaging": false, "translator": { "type": "TabularTranslator" }, "typeConversionSettings": { "typeConversion": { "allowDataTruncation": true, "treatBooleanAsNumber": false } } } }, { "id": "10390776-5fb0-4454-bcbe-b6d1cffc2da8", "properties": { "source": { "datasetSettings": { "schema": "dbo", "table": "fact_sale" } }, "destination": { "partitionOption": "None", "writeBehavior": "Overwrite", "datasetSettings": { "schema": "dbo", "table": "fact_sale" } }, "enableStaging": false, "translator": { "type": "TabularTranslator" }, "typeConversionSettings": { "typeConversion": { "allowDataTruncation": true, "treatBooleanAsNumber": false } } } } ] } ================================================ FILE: sample/workspace/Hello Dataflow.Dataflow/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "Dataflow", "displayName": "Hello Dataflow" }, "config": { "version": "2.0", "logicalId": "35869aa1-a7be-8319-4fa2-a84dac178bf1" } } ================================================ FILE: sample/workspace/Hello Dataflow.Dataflow/mashup.pq ================================================ [StagingDefinition = [Kind = "FastCopy"]] section Section1; shared Table = let Source = Table.FromRows(Json.Document(Binary.Decompress(Binary.FromText("i45W8ssvSU3Kz89W0lEyBGKP1JycfIXw/KKcFKVYnWgll8SSxLSc/HKglBFMGi6hEJBZkJqTmZcKlDEGYoS24NTcxLySzGQF3/yU1ByglAkQg9kKqEZAzTYFYhc3mL2xAA==", BinaryEncoding.Base64), Compression.Deflate)), let _t = ((type nullable text) meta [Serialized.Text = true]) in type table [Item = _t, Id = _t, Name = _t]), #"Changed column type" = Table.TransformColumnTypes(Source, {{"Item", type text}, {"Id", Int64.Type}, {"Name", type text}}), #"Added custom" = Table.TransformColumnTypes(Table.AddColumn(#"Changed column type", "IsDataflow", each if [Item] = "Dataflow" then true else false), {{"IsDataflow", type logical}}), #"Filtered rows" = Table.SelectRows(#"Added custom", each ([IsDataflow] = true)) in #"Filtered rows"; ================================================ FILE: sample/workspace/Hello Dataflow.Dataflow/queryMetadata.json ================================================ { "formatVersion": "202502", "computeEngineSettings": {}, "name": "Hello Dataflow", "queryGroups": [], "documentLocale": "en-US", "queriesMetadata": { "Table": { "queryId": "4f5f3e2c-e1c3-4abe-825e-c98ea699ac8f", "queryName": "Table" } }, "connections": [] } ================================================ FILE: sample/workspace/Hello World.Notebook/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "Notebook", "displayName": "Hello World", "description": "Sample notebook" }, "config": { "version": "2.0", "logicalId": "99b570c5-0c79-9dc4-4c9b-fa16c621384c" } } ================================================ FILE: sample/workspace/Hello World.Notebook/notebook-content.py ================================================ # Fabric notebook source # METADATA ******************** # META { # META "kernel_info": { # META "name": "synapse_pyspark" # META }, # META "dependencies": { # META "environment": { # META "environmentId": "a277ea4a-e87f-8537-4ce0-39db11d4aade", # META "workspaceId": "00000000-0000-0000-0000-000000000000" # META } # META } # META } # CELL ******************** print("Hello World") # METADATA ******************** # META { # META "language": "python", # META "language_group": "synapse_pyspark" # META } ================================================ FILE: sample/workspace/Hello db.SQLDatabase/.gitignore ================================================ ## Ignore Visual Studio temporary files, build results, and ## files generated by popular Visual Studio add-ons. ## ## Get latest from `dotnet new gitignore` # dotenv files .env # User-specific files *.rsuser *.suo *.user *.userosscache *.sln.docstates # User-specific files (MonoDevelop/Xamarin Studio) *.userprefs # Mono auto generated files mono_crash.* # Build results [Dd]ebug/ [Dd]ebugPublic/ [Rr]elease/ [Rr]eleases/ x64/ x86/ [Ww][Ii][Nn]32/ [Aa][Rr][Mm]/ [Aa][Rr][Mm]64/ bld/ [Bb]in/ [Oo]bj/ [Ll]og/ [Ll]ogs/ # Visual Studio 2015/2017 cache/options directory .vs/ # Uncomment if you have tasks that create the project's static files in wwwroot #wwwroot/ # Visual Studio 2017 auto generated files Generated\ Files/ # MSTest test Results [Tt]est[Rr]esult*/ [Bb]uild[Ll]og.* # NUnit *.VisualState.xml TestResult.xml nunit-*.xml # Build Results of an ATL Project [Dd]ebugPS/ [Rr]eleasePS/ dlldata.c # Benchmark Results BenchmarkDotNet.Artifacts/ # .NET project.lock.json project.fragment.lock.json artifacts/ # Tye .tye/ # ASP.NET Scaffolding ScaffoldingReadMe.txt # StyleCop StyleCopReport.xml # Files built by Visual Studio *_i.c *_p.c *_h.h *.ilk *.meta *.obj *.iobj *.pch *.pdb *.ipdb *.pgc *.pgd *.rsp *.sbr *.tlb *.tli *.tlh *.tmp *.tmp_proj *_wpftmp.csproj *.log *.tlog *.vspscc *.vssscc .builds *.pidb *.svclog *.scc # Chutzpah Test files _Chutzpah* # Visual C++ cache files ipch/ *.aps *.ncb *.opendb *.opensdf *.sdf *.cachefile *.VC.db *.VC.VC.opendb # Visual Studio profiler *.psess *.vsp *.vspx *.sap # Visual Studio Trace Files *.e2e # TFS 2012 Local Workspace $tf/ # Guidance Automation Toolkit *.gpState # ReSharper is a .NET coding add-in _ReSharper*/ *.[Rr]e[Ss]harper *.DotSettings.user # TeamCity is a build add-in _TeamCity* # DotCover is a Code Coverage Tool *.dotCover # AxoCover is a Code Coverage Tool .axoCover/* !.axoCover/settings.json # Coverlet is a free, cross platform Code Coverage Tool coverage*.json coverage*.xml coverage*.info # Visual Studio code coverage results *.coverage *.coveragexml # NCrunch _NCrunch_* .*crunch*.local.xml nCrunchTemp_* # MightyMoose *.mm.* AutoTest.Net/ # Web workbench (sass) .sass-cache/ # Installshield output folder [Ee]xpress/ # DocProject is a documentation generator add-in DocProject/buildhelp/ DocProject/Help/*.HxT DocProject/Help/*.HxC DocProject/Help/*.hhc DocProject/Help/*.hhk DocProject/Help/*.hhp DocProject/Help/Html2 DocProject/Help/html # Click-Once directory publish/ # Publish Web Output *.[Pp]ublish.xml *.azurePubxml # Note: Comment the next line if you want to checkin your web deploy settings, # but database connection strings (with potential passwords) will be unencrypted *.pubxml *.publishproj # Microsoft Azure Web App publish settings. Comment the next line if you want to # checkin your Azure Web App publish settings, but sensitive information contained # in these scripts will be unencrypted PublishScripts/ # NuGet Packages *.nupkg # NuGet Symbol Packages *.snupkg # The packages folder can be ignored because of Package Restore **/[Pp]ackages/* # except build/, which is used as an MSBuild target. !**/[Pp]ackages/build/ # Uncomment if necessary however generally it will be regenerated when needed #!**/[Pp]ackages/repositories.config # NuGet v3's project.json files produces more ignorable files *.nuget.props *.nuget.targets # Microsoft Azure Build Output csx/ *.build.csdef # Microsoft Azure Emulator ecf/ rcf/ # Windows Store app package directories and files AppPackages/ BundleArtifacts/ Package.StoreAssociation.xml _pkginfo.txt *.appx *.appxbundle *.appxupload # Visual Studio cache files # files ending in .cache can be ignored *.[Cc]ache # but keep track of directories ending in .cache !?*.[Cc]ache/ # Others ClientBin/ ~$* *~ *.dbmdl *.dbproj.schemaview *.jfm *.pfx *.publishsettings orleans.codegen.cs # Including strong name files can present a security risk # (https://github.com/github/gitignore/pull/2483#issue-259490424) #*.snk # Since there are multiple workflows, uncomment next line to ignore bower_components # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) #bower_components/ # RIA/Silverlight projects Generated_Code/ # Backup & report files from converting an old project file # to a newer Visual Studio version. Backup files are not needed, # because we have git ;-) _UpgradeReport_Files/ Backup*/ UpgradeLog*.XML UpgradeLog*.htm ServiceFabricBackup/ *.rptproj.bak # SQL Server files *.mdf *.ldf *.ndf # Business Intelligence projects *.rdl.data *.bim.layout *.bim_*.settings *.rptproj.rsuser *- [Bb]ackup.rdl *- [Bb]ackup ([0-9]).rdl *- [Bb]ackup ([0-9][0-9]).rdl # Microsoft Fakes FakesAssemblies/ # GhostDoc plugin setting file *.GhostDoc.xml # Node.js Tools for Visual Studio .ntvs_analysis.dat node_modules/ # Visual Studio 6 build log *.plg # Visual Studio 6 workspace options file *.opt # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) *.vbw # Visual Studio 6 auto-generated project file (contains which files were open etc.) *.vbp # Visual Studio 6 workspace and project file (working project files containing files to include in project) *.dsw *.dsp # Visual Studio 6 technical files *.ncb *.aps # Visual Studio LightSwitch build output **/*.HTMLClient/GeneratedArtifacts **/*.DesktopClient/GeneratedArtifacts **/*.DesktopClient/ModelManifest.xml **/*.Server/GeneratedArtifacts **/*.Server/ModelManifest.xml _Pvt_Extensions # Paket dependency manager .paket/paket.exe paket-files/ # FAKE - F# Make .fake/ # CodeRush personal settings .cr/personal # Python Tools for Visual Studio (PTVS) __pycache__/ *.pyc # Cake - Uncomment if you are using it # tools/** # !tools/packages.config # Tabs Studio *.tss # Telerik's JustMock configuration file *.jmconfig # BizTalk build output *.btp.cs *.btm.cs *.odx.cs *.xsd.cs # OpenCover UI analysis results OpenCover/ # Azure Stream Analytics local run output ASALocalRun/ # MSBuild Binary and Structured Log *.binlog # NVidia Nsight GPU debugger configuration file *.nvuser # MFractors (Xamarin productivity tool) working folder .mfractor/ # Local History for Visual Studio .localhistory/ # Visual Studio History (VSHistory) files .vshistory/ # BeatPulse healthcheck temp database healthchecksdb # Backup folder for Package Reference Convert tool in Visual Studio 2017 MigrationBackup/ # Ionide (cross platform F# VS Code tools) working folder .ionide/ # Fody - auto-generated XML schema FodyWeavers.xsd # VS Code files for those working on multiple tools .vscode/* !.vscode/settings.json !.vscode/tasks.json !.vscode/launch.json !.vscode/extensions.json *.code-workspace # Local History for Visual Studio Code .history/ # Windows Installer files from build outputs *.cab *.msi *.msix *.msm *.msp # JetBrains Rider *.sln.iml .idea ## ## Visual studio for Mac ## # globs Makefile.in *.userprefs *.usertasks config.make config.status aclocal.m4 install-sh autom4te.cache/ *.tar.gz tarballs/ test-results/ # Mac bundle stuff *.dmg *.app # content below from: https://github.com/github/gitignore/blob/master/Global/macOS.gitignore # General .DS_Store .AppleDouble .LSOverride # Icon must end with two \r Icon # Thumbnails ._* # Files that might appear in the root of a volume .DocumentRevisions-V100 .fseventsd .Spotlight-V100 .TemporaryItems .Trashes .VolumeIcon.icns .com.apple.timemachine.donotpresent # Directories potentially created on remote AFP share .AppleDB .AppleDesktop Network Trash Folder Temporary Items .apdisk # content below from: https://github.com/github/gitignore/blob/master/Global/Windows.gitignore # Windows thumbnail cache files Thumbs.db ehthumbs.db ehthumbs_vista.db # Dump file *.stackdump # Folder config file [Dd]esktop.ini # Recycle Bin used on file shares $RECYCLE.BIN/ # Windows Installer files *.cab *.msi *.msix *.msm *.msp # Windows shortcuts *.lnk # Vim temporary swap files *.swp ================================================ FILE: sample/workspace/Hello db.SQLDatabase/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "SQLDatabase", "displayName": "Hello db" }, "config": { "version": "2.0", "logicalId": "fab68254-2c13-9624-451f-7ef2912347a6" } } ================================================ FILE: sample/workspace/Hello db.SQLDatabase/Hello db.sqlproj ================================================ Hello db {00000000-0000-0000-0000-000000000000} Microsoft.Data.Tools.Schema.Sql.SqlDbFabricDatabaseSchemaProvider 1033, CI ================================================ FILE: sample/workspace/HelloEventhouse.Eventhouse/.children/HelloEventhouse.KQLDatabase/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "KQLDatabase", "displayName": "HelloEventhouse", "description": "HelloEventhouse" }, "config": { "version": "2.0", "logicalId": "9ec49357-fad2-8ac5-4295-0dcf7d164fc7" } } ================================================ FILE: sample/workspace/HelloEventhouse.Eventhouse/.children/HelloEventhouse.KQLDatabase/DatabaseProperties.json ================================================ { "databaseType": "ReadWrite", "parentEventhouseItemId": "50adae54-3e43-9fda-464a-757b7f9fb86b", "oneLakeCachingPeriod": "P36500D", "oneLakeStandardStoragePeriod": "P36500D" } ================================================ FILE: sample/workspace/HelloEventhouse.Eventhouse/.children/HelloEventhouse.KQLDatabase/DatabaseSchema.kql ================================================ // KQL script // Use management commands in this script to configure your database items, such as tables, functions, materialized views, and more. .create-merge table YellowTaxi (vendorID:string, tpepPickupDateTime:datetime, tpepDropoffDateTime:datetime, passengerCount:int, tripDistance:real, puLocationId:string, doLocationId:string, startLon:string, startLat:string, endLon:string, endLat:string, rateCodeId:string, storeAndFwdFlag:string, paymentType:string, fareAmount:real, extra:int, mtaTax:real, improvementSurcharge:real, tipAmount:real, tollsAmount:int, totalAmount:real, EventProcessedUtcTime:datetime, PartitionId:string, EventEnqueuedUtcTime:datetime) .create-or-alter table YellowTaxi ingestion json mapping 'YellowTaxi_mapping' ``` [{"Properties":{"Path":"$['vendorID']"},"column":"vendorID","datatype":""},{"Properties":{"Path":"$['tpepPickupDateTime']","Transform":"DateTimeFromUnixMilliseconds"},"column":"tpepPickupDateTime","datatype":""},{"Properties":{"Path":"$['tpepDropoffDateTime']","Transform":"DateTimeFromUnixMilliseconds"},"column":"tpepDropoffDateTime","datatype":""},{"Properties":{"Path":"$['passengerCount']"},"column":"passengerCount","datatype":""},{"Properties":{"Path":"$['tripDistance']"},"column":"tripDistance","datatype":""},{"Properties":{"Path":"$['puLocationId']"},"column":"puLocationId","datatype":""},{"Properties":{"Path":"$['doLocationId']"},"column":"doLocationId","datatype":""},{"Properties":{"Path":"$['startLon']"},"column":"startLon","datatype":""},{"Properties":{"Path":"$['startLat']"},"column":"startLat","datatype":""},{"Properties":{"Path":"$['endLon']"},"column":"endLon","datatype":""},{"Properties":{"Path":"$['endLat']"},"column":"endLat","datatype":""},{"Properties":{"Path":"$['rateCodeId']"},"column":"rateCodeId","datatype":""},{"Properties":{"Path":"$['storeAndFwdFlag']"},"column":"storeAndFwdFlag","datatype":""},{"Properties":{"Path":"$['paymentType']"},"column":"paymentType","datatype":""},{"Properties":{"Path":"$['fareAmount']"},"column":"fareAmount","datatype":""},{"Properties":{"Path":"$['extra']"},"column":"extra","datatype":""},{"Properties":{"Path":"$['mtaTax']"},"column":"mtaTax","datatype":""},{"Properties":{"Path":"$['improvementSurcharge']"},"column":"improvementSurcharge","datatype":""},{"Properties":{"Path":"$['tipAmount']"},"column":"tipAmount","datatype":""},{"Properties":{"Path":"$['tollsAmount']"},"column":"tollsAmount","datatype":""},{"Properties":{"Path":"$['totalAmount']"},"column":"totalAmount","datatype":""},{"Properties":{"Path":"$['EventProcessedUtcTime']"},"column":"EventProcessedUtcTime","datatype":""},{"Properties":{"Path":"$['PartitionId']"},"column":"PartitionId","datatype":""},{"Properties":{"Path":"$['EventEnqueuedUtcTime']"},"column":"EventEnqueuedUtcTime","datatype":""}] ``` ================================================ FILE: sample/workspace/HelloEventhouse.Eventhouse/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "Eventhouse", "displayName": "HelloEventhouse", "description": "HelloEventhouse" }, "config": { "version": "2.0", "logicalId": "50adae54-3e43-9fda-464a-757b7f9fb86b" } } ================================================ FILE: sample/workspace/HelloEventhouse.Eventhouse/EventhouseProperties.json ================================================ {} ================================================ FILE: sample/workspace/HelloRealTimeDashboard.KQLDashboard/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "KQLDashboard", "displayName": "HelloRealTimeDashboard", "description": "HelloRealTimeDashboard" }, "config": { "version": "2.0", "logicalId": "d105911e-7acf-b25d-45a2-0cf90211a6d0" } } ================================================ FILE: sample/workspace/HelloRealTimeDashboard.KQLDashboard/RealTimeDashboard.json ================================================ { "schema_version": "60", "tiles": [ { "id": "bad34d7e-6462-4811-b17a-5ed946fa0bf1", "title": "New tile", "visualType": "table", "pageId": "48f62f8e-cb94-443b-8a98-9dae1edb6a35", "layout": { "x": 0, "y": 0, "width": 15, "height": 8 }, "queryRef": { "kind": "query", "queryId": "8e1eced0-4d8a-43a8-8932-ec36f799b06f" }, "visualOptions": { "table__enableRenderLinks": true, "colorRulesDisabled": true, "colorStyle": "light", "crossFilterDisabled": false, "drillthroughDisabled": false, "crossFilter": [], "drillthrough": [], "table__renderLinks": [], "colorRules": [] } }, { "id": "197cd6d1-d4fb-48c2-8abd-cb293552caa5", "title": "New tile", "visualType": "table", "pageId": "48f62f8e-cb94-443b-8a98-9dae1edb6a35", "layout": { "x": 15, "y": 0, "width": 9, "height": 7 }, "queryRef": { "kind": "query", "queryId": "80edd254-bbbe-4d44-af41-a1bf292ee045" }, "visualOptions": { "table__enableRenderLinks": true, "colorRulesDisabled": true, "colorStyle": "light", "crossFilter": [], "crossFilterDisabled": false, "drillthroughDisabled": false, "drillthrough": [], "table__renderLinks": [], "colorRules": [] } } ], "baseQueries": [ { "id": "c6be13e1-fc14-44ef-a4bf-5670ebb67943", "queryId": "6632d0f7-5d1e-47de-9aca-10c04ecd230b", "variableName": "PassengerCount" } ], "parameters": [ { "kind": "duration", "id": "b9410c12-493a-455c-bac9-7d2a7eecd110", "displayName": "Time range", "description": "", "beginVariableName": "_startTime", "endVariableName": "_endTime", "defaultValue": { "kind": "dynamic", "count": 1, "unit": "hours" }, "showOnPages": { "kind": "all" } }, { "kind": "int", "id": "e8acee0e-b0ce-42a9-b340-bd934d434fd3", "displayName": "PassengerCount", "description": "", "variableName": "PassengerCounts", "selectionType": "scalar", "includeAllOption": false, "defaultValue": { "kind": "value", "value": 1 }, "dataSource": { "kind": "static", "values": [ { "value": 1 } ] }, "showOnPages": { "kind": "all" } } ], "dataSources": [ { "kind": "kusto-trident", "scopeId": "kusto-trident", "clusterUri": "", "database": "9ec49357-fad2-8ac5-4295-0dcf7d164fc7", "name": "HelloEventhouse", "id": "13efefa9-c29f-420a-9f53-834aece3eacf", "workspace": "00000000-0000-0000-0000-000000000000" }, { "kind": "kusto-trident", "scopeId": "kusto-trident", "clusterUri": "https://trd-srpupn7qadyes7d72g.z2.kusto.fabric.microsoft.com", "database": "a65d86d8-ba4d-4fbf-b264-b3a504ced0ab", "name": "Test1", "id": "c33c6d70-d6fa-4d50-a8b4-3c44b969c033", "workspace": "90c87160-6b48-4215-994f-710bb0755246" } ], "pages": [ { "name": "Passengers", "id": "48f62f8e-cb94-443b-8a98-9dae1edb6a35" } ], "queries": [ { "dataSource": { "kind": "inline", "dataSourceId": "13efefa9-c29f-420a-9f53-834aece3eacf" }, "text": "// Please enter your KQL query (Example):\n// \n// | where between (['_startTime'] .. ['_endTime']) // Time range filtering\n// | take 100\nYellowTaxi\n| where passengerCount > 1\n", "id": "8e1eced0-4d8a-43a8-8932-ec36f799b06f", "usedVariables": [] }, { "dataSource": { "kind": "inline", "dataSourceId": "c33c6d70-d6fa-4d50-a8b4-3c44b969c033" }, "text": "// Please enter your KQL query (Example):\n//
\n// | where between (['_startTime'] .. ['_endTime']) // Time range filtering\n// | where isempty(['PassengerCounts']) or == ['PassengerCounts'] // Single selection filtering\n// | take 100\nWeather\n| take 100", "id": "80edd254-bbbe-4d44-af41-a1bf292ee045", "usedVariables": [] }, { "id": "6632d0f7-5d1e-47de-9aca-10c04ecd230b", "dataSource": { "kind": "inline", "dataSourceId": "13efefa9-c29f-420a-9f53-834aece3eacf" }, "text": "// Please enter your KQL query (Example):\n//
\n// | where between (['_startTime'] .. ['_endTime']) // Time range filtering\n// | take 100\nYellowTaxi\n| where passengerCount > 1", "usedVariables": [] } ] } ================================================ FILE: sample/workspace/MirroredDatabase_1.MirroredDatabase/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "MirroredDatabase", "displayName": "MirroredDatabase_1" }, "config": { "version": "2.0", "logicalId": "7b1dcb5e-9744-86a5-447a-67e0a4bef6f8" } } ================================================ FILE: sample/workspace/MirroredDatabase_1.MirroredDatabase/mirroring.json ================================================ { "properties": { "source": { "type": "GenericMirror", "typeProperties": null }, "target": { "type": "MountedRelationalDatabase", "typeProperties": { "format": "Delta", "defaultSchema": "dbo" } } } } ================================================ FILE: sample/workspace/OntologyDataLH.Lakehouse/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "Lakehouse", "displayName": "OntologyDataLH" }, "config": { "version": "2.0", "logicalId": "19b772b2-c916-8ae7-41fa-1fa21fbf1fb0" } } ================================================ FILE: sample/workspace/OntologyDataLH.Lakehouse/alm.settings.json ================================================ { "version": "1.0.1", "objectTypes": [ { "name": "Shortcuts", "state": "Enabled", "subObjectTypes": [ { "name": "Shortcuts.OneLake", "state": "Enabled" }, { "name": "Shortcuts.AdlsGen2", "state": "Enabled" }, { "name": "Shortcuts.Dataverse", "state": "Enabled" }, { "name": "Shortcuts.AmazonS3", "state": "Enabled" }, { "name": "Shortcuts.S3Compatible", "state": "Enabled" }, { "name": "Shortcuts.GoogleCloudStorage", "state": "Enabled" }, { "name": "Shortcuts.AzureBlobStorage", "state": "Enabled" }, { "name": "Shortcuts.OneDriveSharePoint", "state": "Enabled" } ] }, { "name": "DataAccessRoles", "state": "Disabled" } ] } ================================================ FILE: sample/workspace/OntologyDataLH.Lakehouse/lakehouse.metadata.json ================================================ {} ================================================ FILE: sample/workspace/OntologyDataLH.Lakehouse/shortcuts.metadata.json ================================================ [] ================================================ FILE: sample/workspace/RetailSalesOntology.Ontology/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "Ontology", "displayName": "RetailSalesOntology" }, "config": { "version": "2.0", "logicalId": "707f3e76-8ff7-867d-4c53-982a5010d492" } } ================================================ FILE: sample/workspace/RetailSalesOntology.Ontology/EntityTypes/205398164146535/DataBindings/a790fdb3-e356-4f42-acf4-4420557c0fd7.json ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/ontology/dataBinding/1.0.0/schema.json", "id": "a790fdb3-e356-4f42-acf4-4420557c0fd7", "dataBindingConfiguration": { "dataBindingType": "NonTimeSeries", "propertyBindings": [ { "sourceColumnName": "StoreId", "targetPropertyId": "4251100807708466193" }, { "sourceColumnName": "StoreName", "targetPropertyId": "4251100806981390461" }, { "sourceColumnName": "City", "targetPropertyId": "4251100807999019669" }, { "sourceColumnName": "Region", "targetPropertyId": "4251100809740297011" }, { "sourceColumnName": "Latitude", "targetPropertyId": "4251100809564483949" }, { "sourceColumnName": "Longitude", "targetPropertyId": "4251100807720423576" } ], "sourceTableProperties": { "sourceType": "LakehouseTable", "workspaceId": "00000000-0000-0000-0000-000000000000", "itemId": "19b772b2-c916-8ae7-41fa-1fa21fbf1fb0", "sourceTableName": "dimstore", "sourceSchema": null } } } ================================================ FILE: sample/workspace/RetailSalesOntology.Ontology/EntityTypes/205398164146535/definition.json ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/ontology/entityType/1.0.0/schema.json", "id": "205398164146535", "namespace": "usertypes", "baseEntityTypeId": null, "name": "Store", "entityIdParts": [ "4251100807708466193" ], "displayNamePropertyId": null, "namespaceType": "Custom", "visibility": "Visible", "properties": [ { "id": "4251100807708466193", "name": "StoreId", "redefines": null, "baseTypeNamespaceType": null, "valueType": "String" }, { "id": "4251100806981390461", "name": "StoreName", "redefines": null, "baseTypeNamespaceType": null, "valueType": "String" }, { "id": "4251100807999019669", "name": "City", "redefines": null, "baseTypeNamespaceType": null, "valueType": "String" }, { "id": "4251100809740297011", "name": "Region", "redefines": null, "baseTypeNamespaceType": null, "valueType": "String" }, { "id": "4251100809564483949", "name": "Latitude", "redefines": null, "baseTypeNamespaceType": null, "valueType": "Double" }, { "id": "4251100807720423576", "name": "Longitude", "redefines": null, "baseTypeNamespaceType": null, "valueType": "Double" } ], "timeseriesProperties": [] } ================================================ FILE: sample/workspace/RetailSalesOntology.Ontology/EntityTypes/267812974919544/DataBindings/275d574a-4d0d-4935-9cfd-54e59ee36d7f.json ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/ontology/dataBinding/1.0.0/schema.json", "id": "275d574a-4d0d-4935-9cfd-54e59ee36d7f", "dataBindingConfiguration": { "dataBindingType": "NonTimeSeries", "propertyBindings": [ { "sourceColumnName": "ProductId", "targetPropertyId": "4247990175846782469" }, { "sourceColumnName": "ProductName", "targetPropertyId": "4247990172388291209" }, { "sourceColumnName": "Category", "targetPropertyId": "4247990173156094732" }, { "sourceColumnName": "Subcategory", "targetPropertyId": "4247990174411431168" }, { "sourceColumnName": "Brand", "targetPropertyId": "4247990176233268513" } ], "sourceTableProperties": { "sourceType": "LakehouseTable", "workspaceId": "00000000-0000-0000-0000-000000000000", "itemId": "19b772b2-c916-8ae7-41fa-1fa21fbf1fb0", "sourceTableName": "dimproducts", "sourceSchema": null } } } ================================================ FILE: sample/workspace/RetailSalesOntology.Ontology/EntityTypes/267812974919544/definition.json ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/ontology/entityType/1.0.0/schema.json", "id": "267812974919544", "namespace": "usertypes", "baseEntityTypeId": null, "name": "Products", "entityIdParts": [ "4247990175846782469" ], "displayNamePropertyId": null, "namespaceType": "Custom", "visibility": "Visible", "properties": [ { "id": "4247990175846782469", "name": "ProductId", "redefines": null, "baseTypeNamespaceType": null, "valueType": "String" }, { "id": "4247990172388291209", "name": "ProductName", "redefines": null, "baseTypeNamespaceType": null, "valueType": "String" }, { "id": "4247990173156094732", "name": "Category", "redefines": null, "baseTypeNamespaceType": null, "valueType": "String" }, { "id": "4247990174411431168", "name": "Subcategory", "redefines": null, "baseTypeNamespaceType": null, "valueType": "String" }, { "id": "4247990176233268513", "name": "Brand", "redefines": null, "baseTypeNamespaceType": null, "valueType": "String" } ], "timeseriesProperties": [] } ================================================ FILE: sample/workspace/RetailSalesOntology.Ontology/EntityTypes/28747097105824/DataBindings/5f66bbb3-9bb6-415a-9cd9-9d7421d0e553.json ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/ontology/dataBinding/1.0.0/schema.json", "id": "5f66bbb3-9bb6-415a-9cd9-9d7421d0e553", "dataBindingConfiguration": { "dataBindingType": "NonTimeSeries", "propertyBindings": [ { "sourceColumnName": "FreezerId", "targetPropertyId": "4184311779038199817" }, { "sourceColumnName": "Model", "targetPropertyId": "4184311778721622097" }, { "sourceColumnName": "minSafeTempC", "targetPropertyId": "4184311777711646739" }, { "sourceColumnName": "StoreId", "targetPropertyId": "4184311777870346068" } ], "sourceTableProperties": { "sourceType": "LakehouseTable", "workspaceId": "00000000-0000-0000-0000-000000000000", "itemId": "19b772b2-c916-8ae7-41fa-1fa21fbf1fb0", "sourceTableName": "freezer", "sourceSchema": null } } } ================================================ FILE: sample/workspace/RetailSalesOntology.Ontology/EntityTypes/28747097105824/DataBindings/f0767964-7f82-40a1-9ca2-f7e7fe931fcd.json ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/ontology/dataBinding/1.0.0/schema.json", "id": "f0767964-7f82-40a1-9ca2-f7e7fe931fcd", "dataBindingConfiguration": { "dataBindingType": "TimeSeries", "timestampColumnName": "timestamp", "propertyBindings": [ { "sourceColumnName": "timestamp", "targetPropertyId": "4162420792245339175" }, { "sourceColumnName": "storeId", "targetPropertyId": "4184311777870346068" }, { "sourceColumnName": "freezerId", "targetPropertyId": "4184311779038199817" }, { "sourceColumnName": "temperatureC", "targetPropertyId": "4162420789917117594" }, { "sourceColumnName": "humidityPct", "targetPropertyId": "4162420789093079066" }, { "sourceColumnName": "doorOpen", "targetPropertyId": "4162420791058306832" } ], "sourceTableProperties": { "sourceType": "KustoTable", "workspaceId": "00000000-0000-0000-0000-000000000000", "itemId": "971dc008-7a5d-a198-4021-03a663f7d3b7", "clusterUri": "https://trd-frevj0bfsve4ey3cpu.z1.kusto.fabric.microsoft.com", "databaseName": "TelemetryDataEH", "sourceTableName": "FreezerTelemetry" } } } ================================================ FILE: sample/workspace/RetailSalesOntology.Ontology/EntityTypes/28747097105824/definition.json ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/ontology/entityType/1.0.0/schema.json", "id": "28747097105824", "namespace": "usertypes", "baseEntityTypeId": null, "name": "Freezer", "entityIdParts": [ "4184311779038199817" ], "displayNamePropertyId": null, "namespaceType": "Custom", "visibility": "Visible", "properties": [ { "id": "4184311779038199817", "name": "FreezerId", "redefines": null, "baseTypeNamespaceType": null, "valueType": "String" }, { "id": "4184311778721622097", "name": "Model", "redefines": null, "baseTypeNamespaceType": null, "valueType": "String" }, { "id": "4184311777711646739", "name": "minSafeTempC", "redefines": null, "baseTypeNamespaceType": null, "valueType": "String" }, { "id": "4184311777870346068", "name": "StoreId", "redefines": null, "baseTypeNamespaceType": null, "valueType": "String" } ], "timeseriesProperties": [ { "id": "4162420792245339175", "name": "timestamp", "redefines": null, "baseTypeNamespaceType": null, "valueType": "DateTime" }, { "id": "4162420789917117594", "name": "temperatureC", "redefines": null, "baseTypeNamespaceType": null, "valueType": "Double" }, { "id": "4162420789093079066", "name": "humidityPct", "redefines": null, "baseTypeNamespaceType": null, "valueType": "Double" }, { "id": "4162420791058306832", "name": "doorOpen", "redefines": null, "baseTypeNamespaceType": null, "valueType": "BigInt" } ] } ================================================ FILE: sample/workspace/RetailSalesOntology.Ontology/EntityTypes/52068896499199/DataBindings/3d6fc8a5-b442-46b2-9bee-c22d08038f2e.json ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/ontology/dataBinding/1.0.0/schema.json", "id": "3d6fc8a5-b442-46b2-9bee-c22d08038f2e", "dataBindingConfiguration": { "dataBindingType": "NonTimeSeries", "propertyBindings": [ { "sourceColumnName": "SaleId", "targetPropertyId": "4246041465402381596" }, { "sourceColumnName": "SaleDate", "targetPropertyId": "4246041463854715983" }, { "sourceColumnName": "StoreId", "targetPropertyId": "4246041466467721141" }, { "sourceColumnName": "ProductId", "targetPropertyId": "4246041463975607651" }, { "sourceColumnName": "Units", "targetPropertyId": "4246041465946373253" }, { "sourceColumnName": "RevenueUSD", "targetPropertyId": "4246041466087597094" } ], "sourceTableProperties": { "sourceType": "LakehouseTable", "workspaceId": "00000000-0000-0000-0000-000000000000", "itemId": "19b772b2-c916-8ae7-41fa-1fa21fbf1fb0", "sourceTableName": "factsales", "sourceSchema": null } } } ================================================ FILE: sample/workspace/RetailSalesOntology.Ontology/EntityTypes/52068896499199/definition.json ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/ontology/entityType/1.0.0/schema.json", "id": "52068896499199", "namespace": "usertypes", "baseEntityTypeId": null, "name": "SaleEvent", "entityIdParts": [ "4246041465402381596" ], "displayNamePropertyId": null, "namespaceType": "Custom", "visibility": "Visible", "properties": [ { "id": "4246041465402381596", "name": "SaleId", "redefines": null, "baseTypeNamespaceType": null, "valueType": "BigInt" }, { "id": "4246041463854715983", "name": "SaleDate", "redefines": null, "baseTypeNamespaceType": null, "valueType": "DateTime" }, { "id": "4246041466467721141", "name": "StoreId", "redefines": null, "baseTypeNamespaceType": null, "valueType": "String" }, { "id": "4246041463975607651", "name": "ProductId", "redefines": null, "baseTypeNamespaceType": null, "valueType": "String" }, { "id": "4246041465946373253", "name": "Units", "redefines": null, "baseTypeNamespaceType": null, "valueType": "BigInt" }, { "id": "4246041466087597094", "name": "RevenueUSD", "redefines": null, "baseTypeNamespaceType": null, "valueType": "Double" } ], "timeseriesProperties": [] } ================================================ FILE: sample/workspace/RetailSalesOntology.Ontology/RelationshipTypes/4160405290834524422/Contextualizations/088a81ce-2a4c-4dbc-8887-ab6b831fa047.json ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/ontology/contextualization/1.0.0/schema.json", "id": "088a81ce-2a4c-4dbc-8887-ab6b831fa047", "dataBindingTable": { "workspaceId": "00000000-0000-0000-0000-000000000000", "itemId": "19b772b2-c916-8ae7-41fa-1fa21fbf1fb0", "sourceTableName": "freezer", "sourceSchema": null, "sourceType": "LakehouseTable" }, "sourceKeyRefBindings": [ { "sourceColumnName": "StoreId", "targetPropertyId": "4251100807708466193" } ], "targetKeyRefBindings": [ { "sourceColumnName": "FreezerId", "targetPropertyId": "4184311779038199817" } ] } ================================================ FILE: sample/workspace/RetailSalesOntology.Ontology/RelationshipTypes/4160405290834524422/definition.json ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/ontology/relationshipType/1.0.0/schema.json", "namespace": "usertypes", "id": "4160405290834524422", "name": "operates", "namespaceType": "Custom", "source": { "entityTypeId": "205398164146535" }, "target": { "entityTypeId": "28747097105824" } } ================================================ FILE: sample/workspace/RetailSalesOntology.Ontology/RelationshipTypes/4194354367812289411/Contextualizations/7be8afdf-2557-4950-930e-26c762fcb5a5.json ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/ontology/contextualization/1.0.0/schema.json", "id": "7be8afdf-2557-4950-930e-26c762fcb5a5", "dataBindingTable": { "workspaceId": "00000000-0000-0000-0000-000000000000", "itemId": "19b772b2-c916-8ae7-41fa-1fa21fbf1fb0", "sourceTableName": "dimproducts", "sourceSchema": null, "sourceType": "LakehouseTable" }, "sourceKeyRefBindings": [ { "sourceColumnName": "ProductId", "targetPropertyId": "4247990175846782469" } ], "targetKeyRefBindings": [ { "sourceColumnName": "ProductId", "targetPropertyId": "4246041465402381596" } ] } ================================================ FILE: sample/workspace/RetailSalesOntology.Ontology/RelationshipTypes/4194354367812289411/definition.json ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/ontology/relationshipType/1.0.0/schema.json", "namespace": "usertypes", "id": "4194354367812289411", "name": "soldIn", "namespaceType": "Custom", "source": { "entityTypeId": "267812974919544" }, "target": { "entityTypeId": "52068896499199" } } ================================================ FILE: sample/workspace/RetailSalesOntology.Ontology/RelationshipTypes/4244194862547506054/Contextualizations/7f23e2a5-25f6-4c87-8b9a-43b3b9a142ec.json ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/ontology/contextualization/1.0.0/schema.json", "id": "7f23e2a5-25f6-4c87-8b9a-43b3b9a142ec", "dataBindingTable": { "workspaceId": "00000000-0000-0000-0000-000000000000", "itemId": "19b772b2-c916-8ae7-41fa-1fa21fbf1fb0", "sourceTableName": "dimstore", "sourceSchema": null, "sourceType": "LakehouseTable" }, "sourceKeyRefBindings": [ { "sourceColumnName": "StoreId", "targetPropertyId": "4251100807708466193" } ], "targetKeyRefBindings": [ { "sourceColumnName": "StoreId", "targetPropertyId": "4246041465402381596" } ] } ================================================ FILE: sample/workspace/RetailSalesOntology.Ontology/RelationshipTypes/4244194862547506054/definition.json ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/ontology/relationshipType/1.0.0/schema.json", "namespace": "usertypes", "id": "4244194862547506054", "name": "has", "namespaceType": "Custom", "source": { "entityTypeId": "205398164146535" }, "target": { "entityTypeId": "52068896499199" } } ================================================ FILE: sample/workspace/RetailSalesOntology.Ontology/definition.json ================================================ {} ================================================ FILE: sample/workspace/Run Hello World.DataPipeline/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "DataPipeline", "displayName": "Run Hello World" }, "config": { "version": "2.0", "logicalId": "70a8992d-af56-801f-4046-ad0813bac453" } } ================================================ FILE: sample/workspace/Run Hello World.DataPipeline/.schedules ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/schedules/1.0.0/schema.json", "schedules": [ { "enabled": true, "jobType": "Execute", "configuration": { "type": "Cron", "startDateTime": "2025-07-01T12:00:00", "endDateTime": "2029-07-01T12:00:00", "localTimeZoneId": "Pacific Standard Time", "interval": 15 } } ] } ================================================ FILE: sample/workspace/Run Hello World.DataPipeline/pipeline-content.json ================================================ { "properties": { "activities": [ { "type": "TridentNotebook", "typeProperties": { "notebookId": "99b570c5-0c79-9dc4-4c9b-fa16c621384c", "workspaceId": "00000000-0000-0000-0000-000000000000" }, "policy": { "timeout": "0.12:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureInput": false, "secureOutput": false }, "name": "Run Hello World", "dependsOn": [] } ] } } ================================================ FILE: sample/workspace/Sample.GraphQLApi/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "GraphQLApi", "displayName": "Sample" }, "config": { "version": "2.0", "logicalId": "c1e7dc70-264e-928d-43fa-a371558f47e4" } } ================================================ FILE: sample/workspace/Sample.GraphQLApi/graphql-definition.json ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/graphqlApi/definition/1.0.0/schema.json", "datasources": [] } ================================================ FILE: sample/workspace/SampleDataActivator.Reflex/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "Reflex", "displayName": "SampleDataActivator" }, "config": { "version": "2.0", "logicalId": "c3bf82de-14b6-af39-4852-dda67eccd7c0" } } ================================================ FILE: sample/workspace/SampleDataActivator.Reflex/ReflexEntities.json ================================================ [] ================================================ FILE: sample/workspace/SampleDataBuildToolJob.DataBuildToolJob/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "DataBuildToolJob", "displayName": "SampleDataBuildToolJob", "description": "Sample DBT job" }, "config": { "version": "2.0", "logicalId": "6f8a77bf-5ff2-4b65-9223-8c10d4fc6a4b" } } ================================================ FILE: sample/workspace/SampleDataBuildToolJob.DataBuildToolJob/dbt-content.json ================================================ { "project": { "projectType": "OneLake", "folderPath": "dbt" }, "profile": { "profileType": "DataWarehouse", "schema": "analytics_schema", "connectionSettings": { "name": "sample_warehouse", "properties": { "type": "DataWarehouse", "typeProperties": { "workspaceId": "00000000-0000-0000-0000-000000000000", "artifactId": "cccccccc-3333-4444-5555-dddddddddddd", "endPoint": "sampleworkspace-samplewarehouse.datawarehouse.fabric.microsoft.com" } } } }, "command": { "operation": "build", "arguments": { "exclude": "", "threads": 4 } } } ================================================ FILE: sample/workspace/SampleEventhouse.Eventhouse/.children/TaxiDB.KQLDatabase/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "KQLDatabase", "displayName": "TaxiDB" }, "config": { "version": "2.0", "logicalId": "a51e98dd-5993-8e1c-443f-02aa53d4db74" } } ================================================ FILE: sample/workspace/SampleEventhouse.Eventhouse/.children/TaxiDB.KQLDatabase/DatabaseProperties.json ================================================ { "databaseType": "ReadWrite", "parentEventhouseItemId": "959f24d2-d283-ad08-4897-52f5ff52f4d3", "oneLakeCachingPeriod": "P30D", "oneLakeStandardStoragePeriod": "P365D" } ================================================ FILE: sample/workspace/SampleEventhouse.Eventhouse/.children/TaxiDB.KQLDatabase/DatabaseSchema.kql ================================================ // KQL script // Use management commands in this script to configure your database items, such as tables, functions, materialized views, and more. .create-merge table TaxiRaw (VendorID:string, tpep_pickup_datetime:datetime, tpep_dropoff_datetime:datetime, passenger_count:real, trip_distance:real, RatecodeID:real, store_and_fwd_flag:string, PULocationID:string, DOLocationID:string, payment_type:long, fare_amount:real, extra:real, mta_tax:real, tip_amount:real, tolls_amount:real, improvement_surcharge:real, total_amount:real, congestion_surcharge:real, airport_fee:real) with (folder = "", docstring = "") .create-or-alter table TaxiRaw ingestion json mapping 'TaxiRaw_mapping' ``` [{"Properties":{"Path":"$['VendorID']"},"column":"VendorID","datatype":""},{"Properties":{"Path":"$['tpep_pickup_datetime']"},"column":"tpep_pickup_datetime","datatype":""},{"Properties":{"Path":"$['tpep_dropoff_datetime']"},"column":"tpep_dropoff_datetime","datatype":""},{"Properties":{"Path":"$['passenger_count']"},"column":"passenger_count","datatype":""},{"Properties":{"Path":"$['trip_distance']"},"column":"trip_distance","datatype":""},{"Properties":{"Path":"$['RatecodeID']"},"column":"RatecodeID","datatype":""},{"Properties":{"Path":"$['store_and_fwd_flag']"},"column":"store_and_fwd_flag","datatype":""},{"Properties":{"Path":"$['PULocationID']"},"column":"PULocationID","datatype":""},{"Properties":{"Path":"$['DOLocationID']"},"column":"DOLocationID","datatype":""},{"Properties":{"Path":"$['payment_type']"},"column":"payment_type","datatype":""},{"Properties":{"Path":"$['fare_amount']"},"column":"fare_amount","datatype":""},{"Properties":{"Path":"$['extra']"},"column":"extra","datatype":""},{"Properties":{"Path":"$['mta_tax']"},"column":"mta_tax","datatype":""},{"Properties":{"Path":"$['tip_amount']"},"column":"tip_amount","datatype":""},{"Properties":{"Path":"$['tolls_amount']"},"column":"tolls_amount","datatype":""},{"Properties":{"Path":"$['improvement_surcharge']"},"column":"improvement_surcharge","datatype":""},{"Properties":{"Path":"$['total_amount']"},"column":"total_amount","datatype":""},{"Properties":{"Path":"$['congestion_surcharge']"},"column":"congestion_surcharge","datatype":""},{"Properties":{"Path":"$['airport_fee']"},"column":"airport_fee","datatype":""}] ``` .create-merge table ZoneLookup (LocationID:string, Borough:string, Zone:string, service_zone:string) .create-merge table TaxiRecords (VendorID:string, tpep_pickup_datetime:datetime, tpep_dropoff_datetime:datetime, passenger_count:real, trip_distance:real, RatecodeID:real, store_and_fwd_flag:string, PULocationID:string, DOLocationID:string, payment_type:long, fare_amount:real, extra:real, mta_tax:real, tip_amount:real, tolls_amount:real, improvement_surcharge:real, total_amount:real, congestion_surcharge:real, airport_fee:real, DOBourough:string, DOZone:string, DOService_Zone:string, PUBourough:string, PUZone:string, PUService_Zone:string) .create-or-alter function with (skipvalidation = "true") TaxiUpdate() {TaxiRaw | lookup (ZoneLookup | project DOLocationID=LocationID, DOBourough=Borough, DOZone=Zone, DOService_Zone=service_zone) on DOLocationID | lookup (ZoneLookup | project PULocationID=LocationID, PUBourough=Borough, PUZone=Zone, PUService_Zone=service_zone) on PULocationID } .create-or-alter materialized-view TaxiRecordsDedup on table TaxiRecords { TaxiRecords | summarize take_any(*) by VendorID, tpep_pickup_datetime, tpep_dropoff_datetime, PULocationID, DOLocationID, trip_distance } .alter materialized-view TaxiRecordsDedup policy caching hotdata=time(14.00:00:00) hotindex=time(14.00:00:00) .create-or-alter materialized-view TaxiRecordsHourly on materialized-view TaxiRecordsDedup { TaxiRecordsDedup | summarize Avg_Passenger_Count=avg(passenger_count), Avg_Trip_Distance=avg(trip_distance), Avg_Fare_Amount=avg(fare_amount), Avg_Extra=avg(extra), Avg_MTA_Tax=avg(mta_tax), Avg_Tip_Amount=avg(tip_amount), Avg_Tolls_Amount=avg(tolls_amount), Avg_Improvement_Surcharge=avg(improvement_surcharge), Avg_Total_Amount=avg(total_amount), Avg_Congestion_Surcharge=avg(congestion_surcharge), Avg_Airport_Fee=avg(airport_fee) by bin(tpep_pickup_datetime,1h), bin(tpep_dropoff_datetime,1h), store_and_fwd_flag, PULocationID, DOLocationID, payment_type, DOBourough, DOZone, DOService_Zone, PUBourough, PUZone, PUService_Zone } .alter materialized-view TaxiRecordsHourly policy caching hotdata=time(60.00:00:00) hotindex=time(60.00:00:00) .alter table TaxiRaw policy retention @'{"SoftDeletePeriod":"1.00:00:00","Recoverability":"Enabled"}' .alter table TaxiRaw policy caching hotdata = time(1.00:00:00) hotindex = time(1.00:00:00) .alter table TaxiRaw policy streamingingestion "{\"IsEnabled\":false,\"HintAllocatedRate\":null,\"NumberOfRowStores\":null,\"SealIntervalLimit\":null,\"SealThresholdBytes\":null,\"UsageTags\":[],\"IsMaintenanceActive\":false}" .alter table TaxiRecords policy retention @'{"SoftDeletePeriod":"10.00:00:00","Recoverability":"Enabled"}' .alter table TaxiRecords policy caching hotdata = time(3.00:00:00) hotindex = time(3.00:00:00) .alter table TaxiRecords policy update "[{\"IsEnabled\":true,\"Source\":\"TaxiRaw\",\"Query\":\"TaxiUpdate()\",\"IsTransactional\":true,\"PropagateIngestionProperties\":false,\"ManagedIdentity\":null}]" ================================================ FILE: sample/workspace/SampleEventhouse.Eventhouse/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "Eventhouse", "displayName": "SampleEventhouse" }, "config": { "version": "2.0", "logicalId": "959f24d2-d283-ad08-4897-52f5ff52f4d3" } } ================================================ FILE: sample/workspace/SampleEventhouse.Eventhouse/EventhouseProperties.json ================================================ {} ================================================ FILE: sample/workspace/SampleEventstream.Eventstream/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "Eventstream", "displayName": "SampleEventstream" }, "config": { "version": "2.0", "logicalId": "5d252c47-5ea6-b389-447d-d5f9aec1c6e7" } } ================================================ FILE: sample/workspace/SampleEventstream.Eventstream/eventstream.json ================================================ { "sources": [ { "id": "d4686b8c-e228-4328-871d-1f2d2bcd6248", "name": "Taxi", "type": "SampleData", "properties": { "type": "YellowTaxi" } } ], "destinations": [ { "id": "07890dca-ce9e-4cb0-ad46-c4a6ee442119", "name": "DataActivator", "type": "Activator", "properties": { "workspaceId": "00000000-0000-0000-0000-000000000000", "itemId": "c3bf82de-14b6-af39-4852-dda67eccd7c0", "inputSerialization": { "type": "Json", "properties": { "encoding": "UTF8" } } }, "inputNodes": [ { "name": "ManageFields" } ], "inputSchemas": [ { "name": "ManageFields", "schema": { "columns": [ { "name": "VendorID", "type": "Nvarchar(max)", "fields": null, "items": null }, { "name": "tpep_pickup_datetime", "type": "DateTime", "fields": null, "items": null }, { "name": "tpep_dropoff_datetime", "type": "DateTime", "fields": null, "items": null }, { "name": "passenger_count", "type": "Float", "fields": null, "items": null }, { "name": "trip_distance", "type": "Float", "fields": null, "items": null }, { "name": "RatecodeID", "type": "Float", "fields": null, "items": null }, { "name": "store_and_fwd_flag", "type": "Nvarchar(max)", "fields": null, "items": null }, { "name": "PULocationID", "type": "Nvarchar(max)", "fields": null, "items": null }, { "name": "DOLocationID", "type": "Nvarchar(max)", "fields": null, "items": null }, { "name": "payment_type", "type": "BigInt", "fields": null, "items": null }, { "name": "fare_amount", "type": "Float", "fields": null, "items": null }, { "name": "extra", "type": "Float", "fields": null, "items": null }, { "name": "mta_tax", "type": "Float", "fields": null, "items": null }, { "name": "tip_amount", "type": "Float", "fields": null, "items": null }, { "name": "tolls_amount", "type": "Float", "fields": null, "items": null }, { "name": "improvement_surcharge", "type": "Float", "fields": null, "items": null }, { "name": "total_amount", "type": "Float", "fields": null, "items": null }, { "name": "congestion_surcharge", "type": "Float", "fields": null, "items": null }, { "name": "airport_fee", "type": "Float", "fields": null, "items": null } ] } } ] }, { "id": "124fb4cb-6c67-4c0a-ab0a-8a99bfefcdca", "name": "Lakehouse", "type": "Lakehouse", "properties": { "workspaceId": "00000000-0000-0000-0000-000000000000", "itemId": "c916eeb0-dd6a-ae32-4f4f-966d2414b239", "schema": "", "deltaTable": "Taxi", "inputSerialization": { "type": "Json", "properties": { "encoding": "UTF8" } } }, "inputNodes": [ { "name": "ManageFields" } ], "inputSchemas": [ { "name": "ManageFields", "schema": { "columns": [ { "name": "VendorID", "type": "Nvarchar(max)", "fields": null, "items": null }, { "name": "tpep_pickup_datetime", "type": "DateTime", "fields": null, "items": null }, { "name": "tpep_dropoff_datetime", "type": "DateTime", "fields": null, "items": null }, { "name": "passenger_count", "type": "Float", "fields": null, "items": null }, { "name": "trip_distance", "type": "Float", "fields": null, "items": null }, { "name": "RatecodeID", "type": "Float", "fields": null, "items": null }, { "name": "store_and_fwd_flag", "type": "Nvarchar(max)", "fields": null, "items": null }, { "name": "PULocationID", "type": "Nvarchar(max)", "fields": null, "items": null }, { "name": "DOLocationID", "type": "Nvarchar(max)", "fields": null, "items": null }, { "name": "payment_type", "type": "BigInt", "fields": null, "items": null }, { "name": "fare_amount", "type": "Float", "fields": null, "items": null }, { "name": "extra", "type": "Float", "fields": null, "items": null }, { "name": "mta_tax", "type": "Float", "fields": null, "items": null }, { "name": "tip_amount", "type": "Float", "fields": null, "items": null }, { "name": "tolls_amount", "type": "Float", "fields": null, "items": null }, { "name": "improvement_surcharge", "type": "Float", "fields": null, "items": null }, { "name": "total_amount", "type": "Float", "fields": null, "items": null }, { "name": "congestion_surcharge", "type": "Float", "fields": null, "items": null }, { "name": "airport_fee", "type": "Float", "fields": null, "items": null } ] } } ] }, { "id": "cecf7a31-bfba-437a-b878-ac6ddb3d50a2", "name": "Eventhouse", "type": "Eventhouse", "properties": { "dataIngestionMode": "ProcessedIngestion", "workspaceId": "00000000-0000-0000-0000-000000000000", "itemId": "a51e98dd-5993-8e1c-443f-02aa53d4db74", "databaseName": "TaxiDB", "tableName": "TaxiRaw", "inputSerialization": { "type": "Json", "properties": { "encoding": "UTF8" } } }, "inputNodes": [ { "name": "ManageFields" } ], "inputSchemas": [ { "name": "ManageFields", "schema": { "columns": [ { "name": "VendorID", "type": "Nvarchar(max)", "fields": null, "items": null }, { "name": "tpep_pickup_datetime", "type": "DateTime", "fields": null, "items": null }, { "name": "tpep_dropoff_datetime", "type": "DateTime", "fields": null, "items": null }, { "name": "passenger_count", "type": "Float", "fields": null, "items": null }, { "name": "trip_distance", "type": "Float", "fields": null, "items": null }, { "name": "RatecodeID", "type": "Float", "fields": null, "items": null }, { "name": "store_and_fwd_flag", "type": "Nvarchar(max)", "fields": null, "items": null }, { "name": "PULocationID", "type": "Nvarchar(max)", "fields": null, "items": null }, { "name": "DOLocationID", "type": "Nvarchar(max)", "fields": null, "items": null }, { "name": "payment_type", "type": "BigInt", "fields": null, "items": null }, { "name": "fare_amount", "type": "Float", "fields": null, "items": null }, { "name": "extra", "type": "Float", "fields": null, "items": null }, { "name": "mta_tax", "type": "Float", "fields": null, "items": null }, { "name": "tip_amount", "type": "Float", "fields": null, "items": null }, { "name": "tolls_amount", "type": "Float", "fields": null, "items": null }, { "name": "improvement_surcharge", "type": "Float", "fields": null, "items": null }, { "name": "total_amount", "type": "Float", "fields": null, "items": null }, { "name": "congestion_surcharge", "type": "Float", "fields": null, "items": null }, { "name": "airport_fee", "type": "Float", "fields": null, "items": null } ] } } ] } ], "streams": [ { "id": "bc291021-eb52-49b7-8c15-01bb51d8a940", "name": "Taxi-Stream", "type": "DefaultStream", "properties": {}, "inputNodes": [ { "name": "Taxi" } ] } ], "operators": [ { "name": "ManageFields", "type": "ManageFields", "inputNodes": [ { "name": "Taxi-Stream" } ], "properties": { "columns": [ { "type": "Rename", "properties": { "column": { "expressionType": "ColumnReference", "node": null, "columnName": "VendorID", "columnPathSegments": [] } }, "alias": "VendorID" }, { "type": "Cast", "properties": { "targetDataType": "DateTime", "column": { "expressionType": "ColumnReference", "node": null, "columnName": "tpep_pickup_datetime", "columnPathSegments": [] } }, "alias": "tpep_pickup_datetime" }, { "type": "Cast", "properties": { "targetDataType": "DateTime", "column": { "expressionType": "ColumnReference", "node": null, "columnName": "tpep_dropoff_datetime", "columnPathSegments": [] } }, "alias": "tpep_dropoff_datetime" }, { "type": "Cast", "properties": { "targetDataType": "Float", "column": { "expressionType": "ColumnReference", "node": null, "columnName": "passenger_count", "columnPathSegments": [] } }, "alias": "passenger_count" }, { "type": "Cast", "properties": { "targetDataType": "Float", "column": { "expressionType": "ColumnReference", "node": null, "columnName": "trip_distance", "columnPathSegments": [] } }, "alias": "trip_distance" }, { "type": "Cast", "properties": { "targetDataType": "Float", "column": { "expressionType": "ColumnReference", "node": null, "columnName": "RatecodeID", "columnPathSegments": [] } }, "alias": "RatecodeID" }, { "type": "Rename", "properties": { "column": { "expressionType": "ColumnReference", "node": null, "columnName": "store_and_fwd_flag", "columnPathSegments": [] } }, "alias": "store_and_fwd_flag" }, { "type": "Rename", "properties": { "column": { "expressionType": "ColumnReference", "node": null, "columnName": "PULocationID", "columnPathSegments": [] } }, "alias": "PULocationID" }, { "type": "Rename", "properties": { "column": { "expressionType": "ColumnReference", "node": null, "columnName": "DOLocationID", "columnPathSegments": [] } }, "alias": "DOLocationID" }, { "type": "Cast", "properties": { "targetDataType": "BigInt", "column": { "expressionType": "ColumnReference", "node": null, "columnName": "payment_type", "columnPathSegments": [] } }, "alias": "payment_type" }, { "type": "Cast", "properties": { "targetDataType": "Float", "column": { "expressionType": "ColumnReference", "node": null, "columnName": "fare_amount", "columnPathSegments": [] } }, "alias": "fare_amount" }, { "type": "Cast", "properties": { "targetDataType": "Float", "column": { "expressionType": "ColumnReference", "node": null, "columnName": "extra", "columnPathSegments": [] } }, "alias": "extra" }, { "type": "Cast", "properties": { "targetDataType": "Float", "column": { "expressionType": "ColumnReference", "node": null, "columnName": "mta_tax", "columnPathSegments": [] } }, "alias": "mta_tax" }, { "type": "Cast", "properties": { "targetDataType": "Float", "column": { "expressionType": "ColumnReference", "node": null, "columnName": "tip_amount", "columnPathSegments": [] } }, "alias": "tip_amount" }, { "type": "Cast", "properties": { "targetDataType": "Float", "column": { "expressionType": "ColumnReference", "node": null, "columnName": "tolls_amount", "columnPathSegments": [] } }, "alias": "tolls_amount" }, { "type": "Cast", "properties": { "targetDataType": "Float", "column": { "expressionType": "ColumnReference", "node": null, "columnName": "improvement_surcharge", "columnPathSegments": [] } }, "alias": "improvement_surcharge" }, { "type": "Cast", "properties": { "targetDataType": "Float", "column": { "expressionType": "ColumnReference", "node": null, "columnName": "total_amount", "columnPathSegments": [] } }, "alias": "total_amount" }, { "type": "Cast", "properties": { "targetDataType": "Float", "column": { "expressionType": "ColumnReference", "node": null, "columnName": "congestion_surcharge", "columnPathSegments": [] } }, "alias": "congestion_surcharge" }, { "type": "Cast", "properties": { "targetDataType": "Float", "column": { "expressionType": "ColumnReference", "node": null, "columnName": "airport_fee", "columnPathSegments": [] } }, "alias": "airport_fee" } ] }, "inputSchemas": [ { "name": "Taxi-Stream", "schema": { "columns": [ { "name": "VendorID", "type": "Nvarchar(max)", "fields": null, "items": null }, { "name": "tpep_pickup_datetime", "type": "Nvarchar(max)", "fields": null, "items": null }, { "name": "tpep_dropoff_datetime", "type": "Nvarchar(max)", "fields": null, "items": null }, { "name": "passenger_count", "type": "Nvarchar(max)", "fields": null, "items": null }, { "name": "trip_distance", "type": "Nvarchar(max)", "fields": null, "items": null }, { "name": "RatecodeID", "type": "Nvarchar(max)", "fields": null, "items": null }, { "name": "store_and_fwd_flag", "type": "Nvarchar(max)", "fields": null, "items": null }, { "name": "PULocationID", "type": "Nvarchar(max)", "fields": null, "items": null }, { "name": "DOLocationID", "type": "Nvarchar(max)", "fields": null, "items": null }, { "name": "payment_type", "type": "Nvarchar(max)", "fields": null, "items": null }, { "name": "fare_amount", "type": "Nvarchar(max)", "fields": null, "items": null }, { "name": "extra", "type": "Nvarchar(max)", "fields": null, "items": null }, { "name": "mta_tax", "type": "Nvarchar(max)", "fields": null, "items": null }, { "name": "tip_amount", "type": "Nvarchar(max)", "fields": null, "items": null }, { "name": "tolls_amount", "type": "Nvarchar(max)", "fields": null, "items": null }, { "name": "improvement_surcharge", "type": "Nvarchar(max)", "fields": null, "items": null }, { "name": "total_amount", "type": "Nvarchar(max)", "fields": null, "items": null }, { "name": "congestion_surcharge", "type": "Nvarchar(max)", "fields": null, "items": null }, { "name": "airport_fee", "type": "Nvarchar(max)", "fields": null, "items": null } ] } } ] } ], "compatibilityLevel": "1.0" } ================================================ FILE: sample/workspace/SampleEventstream.Eventstream/eventstreamProperties.json ================================================ { "retentionTimeInDays": 1, "eventThroughputLevel": "Low" } ================================================ FILE: sample/workspace/SampleKQLQueryset.KQLQueryset/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "KQLQueryset", "displayName": "SampleKQLQueryset", "description": "Sample" }, "config": { "version": "2.0", "logicalId": "6f343e19-fcaa-a492-4d58-6bd62a31fb94" } } ================================================ FILE: sample/workspace/SampleKQLQueryset.KQLQueryset/RealTimeQueryset.json ================================================ { "queryset": { "version": "1.0.0", "dataSources": [ { "id": "50c1256c-4b67-4f03-a048-9aeadd277887", "clusterUri": "", "type": "Fabric", "databaseItemId": "a51e98dd-5993-8e1c-443f-02aa53d4db74", "databaseItemName": "TaxiDB" } ], "tabs": [ { "id": "472f60f7-4cdb-4cb2-a2fb-a7fce8377aa0", "content": "//***********************************************************************************************************\n// Here are two articles to help you get started with KQL:\n// KQL reference guide - https://aka.ms/KQLguide\n// SQL - KQL conversions - https://aka.ms/sqlcheatsheet\n//***********************************************************************************************************\n\n// Use \"take\" to view a sample number of records in the table and check the data.\nTaxiRaw\n| take 100\n\n// See how many records are in the table.\nTaxiRaw\n| count\n\n// This query returns the number of ingestions per hour in the given table.\nTaxiRecords\n| summarize IngestionCount = count() by bin(ingestion_time(), 1h)\n\n", "title": "", "dataSourceId": "50c1256c-4b67-4f03-a048-9aeadd277887" } ] } } ================================================ FILE: sample/workspace/SampleSparkJobDefinition.SparkJobDefinition/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "SparkJobDefinition", "displayName": "SampleSparkJobDefinition", "description": "Spark job definition" }, "config": { "version": "2.0", "logicalId": "674fe1e5-9c35-8b9d-48cd-98a3fd65b0b2" } } ================================================ FILE: sample/workspace/SampleSparkJobDefinition.SparkJobDefinition/Libs/pipeline_config.py ================================================ config = { 'range_limit': 1000 } ================================================ FILE: sample/workspace/SampleSparkJobDefinition.SparkJobDefinition/Main/main.py ================================================ from pyspark.sql import SparkSession import logging import sys from pipeline_config import config # Configure logging logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', handlers=[logging.StreamHandler(sys.stdout)] ) logger = logging.getLogger(__name__) if __name__ == "__main__": # Step 1: Create Spark Session spark = (SparkSession .builder .appName("sjdsampleapp") .getOrCreate()) spark_context = spark.sparkContext spark_context.setLogLevel("ERROR") logger.info("=" * 80) logger.info("Starting Job") logger.info("=" * 80) # Step 2: Sample Data Processing df = spark.range(config['range_limit']).collect() logger.info("=" * 80) logger.info("Completing Job") logger.info("=" * 80) ================================================ FILE: sample/workspace/SampleSparkJobDefinition.SparkJobDefinition/SparkJobDefinitionV1.json ================================================ { "executableFile": "main.py", "defaultLakehouseArtifactId": "eb8e6aef-81f9-894c-4eda-7be021fdfc5d", "mainClass": "", "additionalLakehouseIds": [], "retryPolicy": null, "commandLineArguments": "arg1 true", "additionalLibraryUris": ["pipeline_config.py"], "language": "Python", "environmentArtifactId": "a277ea4a-e87f-8537-4ce0-39db11d4aade" } ================================================ FILE: sample/workspace/SampleUserDataFunction.UserDataFunction/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "UserDataFunction", "displayName": "SampleUserDataFunction" }, "config": { "version": "2.0", "logicalId": "ee3a4088-9b35-8f20-4977-ffbae687acc4" } } ================================================ FILE: sample/workspace/SampleUserDataFunction.UserDataFunction/.resources/functions.json ================================================ { "runtime": "PYTHON", "functionsMetadata": [ { "name": "hello_fabric", "scriptFile": "function_app.py", "bindings": [ { "name": "req", "type": "HttpTrigger", "direction": "In", "authLevel": "Anonymous", "methods": [ "post" ], "route": "hello_fabric" } ], "fabricProperties": { "fabricMetadataSchemaVersion": null, "fabricFunctionReturnType": "str", "fabricFunctionParameters": [ { "name": "name", "dataType": "str" } ] } } ] } ================================================ FILE: sample/workspace/SampleUserDataFunction.UserDataFunction/definition.json ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/userDataFunction/definition/1.1.0/schema.json", "runtime": "PYTHON", "connectedDataSources": [], "functions": [ { "name": "hello_fabric", "description": "", "isPublicEndpointEnabled": true } ], "libraries": { "public": [ { "name": "fabric-user-data-functions", "type": "PYPI", "version": "1.0" } ], "private": [] } } ================================================ FILE: sample/workspace/SampleUserDataFunction.UserDataFunction/function_app.py ================================================ import datetime import fabric.functions as fn import logging udf = fn.UserDataFunctions() @udf.function() def hello_fabric(name: str) -> str: logging.info('Python UDF trigger function processed a request.') return f"Welcome to Fabric Functions, {name}, at {datetime.datetime.now()}!" ================================================ FILE: sample/workspace/SourceForShortcutLH.Lakehouse/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "Lakehouse", "displayName": "SourceForShortcutLH" }, "config": { "version": "2.0", "logicalId": "c0cec3c6-bff6-8a77-47b1-ab19d15a52cc" } } ================================================ FILE: sample/workspace/SourceForShortcutLH.Lakehouse/lakehouse.metadata.json ================================================ {"defaultSchema":"dbo"} ================================================ FILE: sample/workspace/SourceForShortcutLH.Lakehouse/shortcuts.metadata.json ================================================ [] ================================================ FILE: sample/workspace/TargetForShortcutLH.Lakehouse/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "Lakehouse", "displayName": "TargetForShortcutLH" }, "config": { "version": "2.0", "logicalId": "30cba7ea-14f6-810a-4c89-6ab6f4632da6" } } ================================================ FILE: sample/workspace/TargetForShortcutLH.Lakehouse/lakehouse.metadata.json ================================================ {"defaultSchema":"dbo"} ================================================ FILE: sample/workspace/TargetForShortcutLH.Lakehouse/shortcuts.metadata.json ================================================ [ { "name": "publicholidays", "path": "/Tables/dbo", "target": { "type": "OneLake", "oneLake": { "path": "Tables/dbo/publicholidays", "itemId": "c0cec3c6-bff6-8a77-47b1-ab19d15a52cc", "workspaceId": "00000000-0000-0000-0000-000000000000", "artifactType": "Lakehouse" } } }, { "name": "sample_datasets", "path": "/Files", "target": { "type": "OneLake", "oneLake": { "path": "Files/sample_datasets", "itemId": "c0cec3c6-bff6-8a77-47b1-ab19d15a52cc", "workspaceId": "00000000-0000-0000-0000-000000000000", "artifactType": "Lakehouse" } } }, { "name": "images", "path": "/Files", "target": { "type": "OneLake", "oneLake": { "path": "Files/images", "itemId": "c0cec3c6-bff6-8a77-47b1-ab19d15a52cc", "workspaceId": "00000000-0000-0000-0000-000000000000", "artifactType": "Lakehouse" } } } ] ================================================ FILE: sample/workspace/TelemetryDataEH.Eventhouse/.children/TelemetryDataEH.KQLDatabase/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "KQLDatabase", "displayName": "TelemetryDataEH", "description": "TelemetryDataEH" }, "config": { "version": "2.0", "logicalId": "65341627-8b62-85ef-49a4-65cc214b4411" } } ================================================ FILE: sample/workspace/TelemetryDataEH.Eventhouse/.children/TelemetryDataEH.KQLDatabase/DatabaseProperties.json ================================================ { "databaseType": "ReadWrite", "parentEventhouseItemId": "971dc008-7a5d-a198-4021-03a663f7d3b7", "oneLakeCachingPeriod": "P36500D", "oneLakeStandardStoragePeriod": "P36500D" } ================================================ FILE: sample/workspace/TelemetryDataEH.Eventhouse/.children/TelemetryDataEH.KQLDatabase/DatabaseSchema.kql ================================================ // KQL script // Use management commands in this script to configure your database items, such as tables, functions, materialized views, and more. .create-merge table FreezerTelemetry (timestamp:datetime, storeId:string, freezerId:string, temperatureC:real, humidityPct:real, doorOpen:long) .create-or-alter table FreezerTelemetry ingestion csv mapping 'FreezerTelemetry_mapping' ``` [{"Properties":{"Ordinal":"0"},"column":"timestamp","datatype":""},{"Properties":{"Ordinal":"1"},"column":"storeId","datatype":""},{"Properties":{"Ordinal":"2"},"column":"freezerId","datatype":""},{"Properties":{"Ordinal":"3"},"column":"temperatureC","datatype":""},{"Properties":{"Ordinal":"4"},"column":"humidityPct","datatype":""},{"Properties":{"Ordinal":"5"},"column":"doorOpen","datatype":""}] ``` ================================================ FILE: sample/workspace/TelemetryDataEH.Eventhouse/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "Eventhouse", "displayName": "TelemetryDataEH", "description": "TelemetryDataEH" }, "config": { "version": "2.0", "logicalId": "971dc008-7a5d-a198-4021-03a663f7d3b7" } } ================================================ FILE: sample/workspace/TelemetryDataEH.Eventhouse/EventhouseProperties.json ================================================ {} ================================================ FILE: sample/workspace/Vars.VariableLibrary/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "VariableLibrary", "displayName": "Vars", "description": "" }, "config": { "version": "2.0", "logicalId": "b2380160-e03e-a112-414f-13442f7f96af" } } ================================================ FILE: sample/workspace/Vars.VariableLibrary/settings.json ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/variableLibrary/definition/settings/1.0.0/schema.json", "valueSetsOrder": [ "PPE", "PROD" ] } ================================================ FILE: sample/workspace/Vars.VariableLibrary/valueSets/PPE.json ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/variableLibrary/definition/valueSet/1.0.0/schema.json", "name": "PPE", "variableOverrides": [] } ================================================ FILE: sample/workspace/Vars.VariableLibrary/valueSets/PROD.json ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/variableLibrary/definition/valueSet/1.0.0/schema.json", "name": "PROD", "variableOverrides": [ { "name": "Environment", "value": "Prod" }, { "name": "SQL_Server", "value": "contoso-prod.database.windows.net" } ] } ================================================ FILE: sample/workspace/Vars.VariableLibrary/variables.json ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/variableLibrary/definition/variables/1.0.0/schema.json", "variables": [ { "name": "Environment", "note": "", "type": "String", "value": "PPE" }, { "name": "SQL_Server", "note": "", "type": "String", "value": "contoso-ppe.database.windows.net" } ] } ================================================ FILE: sample/workspace/WithSchema.Lakehouse/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "Lakehouse", "displayName": "WithSchema" }, "config": { "version": "2.0", "logicalId": "eb8e6aef-81f9-894c-4eda-7be021fdfc5d" } } ================================================ FILE: sample/workspace/WithSchema.Lakehouse/lakehouse.metadata.json ================================================ {"defaultSchema":"dbo"} ================================================ FILE: sample/workspace/WithSchema.Lakehouse/shortcuts.metadata.json ================================================ [ { "name": "Testing", "path": "/Files", "target": { "type": "OneLake", "oneLake": { "path": "Files/Testing", "itemId": "c916eeb0-dd6a-ae32-4f4f-966d2414b239", "workspaceId": "00000000-0000-0000-0000-000000000000", "artifactType": "Lakehouse" } } } ] ================================================ FILE: sample/workspace/WithoutSchema.Lakehouse/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "Lakehouse", "displayName": "WithoutSchema" }, "config": { "version": "2.0", "logicalId": "c916eeb0-dd6a-ae32-4f4f-966d2414b239" } } ================================================ FILE: sample/workspace/WithoutSchema.Lakehouse/lakehouse.metadata.json ================================================ {} ================================================ FILE: sample/workspace/WithoutSchema.Lakehouse/shortcuts.metadata.json ================================================ [] ================================================ FILE: sample/workspace/World.Environment/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "Environment", "displayName": "World", "description": "Environment" }, "config": { "version": "2.0", "logicalId": "a277ea4a-e87f-8537-4ce0-39db11d4aade" } } ================================================ FILE: sample/workspace/World.Environment/Libraries/PublicLibraries/environment.yml ================================================ dependencies: - pip: - fuzzywuzzy==0.18.0 - python-levenshtein==0.26.0 ================================================ FILE: sample/workspace/World.Environment/Setting/Sparkcompute.yml ================================================ enable_native_execution_engine: false driver_cores: 8 driver_memory: 56g executor_cores: 8 executor_memory: 56g dynamic_executor_allocation: enabled: true min_executors: 1 max_executors: 9 runtime_version: 1.2 ================================================ FILE: sample/workspace/cicd_experiment.MLExperiment/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "MLExperiment", "displayName": "cicd_experiment" }, "config": { "version": "2.0", "logicalId": "ab6b3e5c-9425-b94c-40be-0ed409f844eb" } } ================================================ FILE: sample/workspace/cicd_experiment.MLExperiment/mlexperiment.metadata.json ================================================ {"dependencies":[]} ================================================ FILE: sample/workspace/config.yml ================================================ # Sample configuration file for fabric-cicd deployment # This file demonstrates the YAML configuration structure for simplified deployment workflow core: # Core configurations # Either workspace or workspace_id must be provided workspace: # Workspace names by environment dev: Fabric-Dev-Engineering test: Fabric-Test-Engineering prod: Fabric-Prod-Engineering workspace_id: # Workspace IDs by environment (takes precedence over workspace if both are provided) dev: 8b6e2c7a-4c1f-4e3a-9b2e-7d8f2e1a6c3b test: 2f4b9e8d-1a7c-4d3e-b8e2-5c9f7a2d4e1b prod: 7c3e1f8b-2d4a-4b9e-8f2c-1a6c3b7d8e2f repository_directory: "." # Path to workspace items directory (relative to config.yml location) (required) item_types_in_scope: # Item types to include in deployment (optional) - VariableLibrary - Dataflow - DataPipeline - Notebook - Environment parameter: "parameter.yml" # Path to parameter file (relative to config.yml location) (optional) publish: # Publish configuration (optional) exclude_regex: "^DONT_DEPLOY.*" # Regex pattern to exclude items from publishing # folder_exclude_regex: "^/DONT_DEPLOY_FOLDER" # Regex pattern to exclude folder paths with items from publishing (requires feature flags) # folder_path_to_include: # Optional list of specific folder paths with items to publish (requires feature flags) # - "/subfolderA" # - "/subfolderA/subfolderB" # items_to_include: # Optional list of specific items to publish (requires feature flags) # - "Hello World.Notebook" # - "Run Hello World.DataPipeline" # shortcut_exclude_regex: "^DONT_DEPLOY_SHORTCUT.*" # Regex pattern to exclude Lakehouse shortcuts from publishing (requires feature flags) skip: # Skip publishing for specific environments dev: true # Skip publishing in dev environment test: false # Enable publishing in test environment prod: false # Enable publishing in prod environment unpublish: # Unpublish configuration (optional) exclude_regex: "^DEBUG.*" # Regex pattern to exclude items from unpublishing # items_to_include: # Optional list of specific items to unpublish (requires feature flags) skip: # Skip unpublishing for specific environments dev: true # Skip unpublishing in dev environment test: false # Enable unpublishing in test environment prod: false # Enable unpublishing in prod environment features: # Feature flags to enable (optional) - enable_shortcut_publish constants: # Global constants to override (optional) DEFAULT_API_ROOT_URL: "https://msitapi.fabric.microsoft.com" ================================================ FILE: sample/workspace/parameter template.yml ================================================ # Add template parameter files extend: - "./templates/nb parameter template 1.yml" - "./templates/nb parameter template 2.yml" find_replace: # Lakehouse Connection Guid - find_value: "db52be81-c2b2-4261-84fa-840c67f4bbd0" replace_value: PPE: "81bbb339-8d0b-46e8-bfa6-289a159c0733" PROD: "5d6a1b16-447f-464a-b959-45d0fed35ca0" # Optional fields: item_type: "Notebook" item_name: ["Hello World", "Hello World Subfolder"] file_path: - "/Hello World.Notebook/notebook-content.py" - "/subfolder/Hello World Subfolder.Notebook/notebook-content.py" ================================================ FILE: sample/workspace/parameter.yml ================================================ find_replace: # Lakehouse Connection Guid - find_value: "db52be81-c2b2-4261-84fa-840c67f4bbd0" replace_value: PPE: "81bbb339-8d0b-46e8-bfa6-289a159c0733" PROD: "5d6a1b16-447f-464a-b959-45d0fed35ca0" # Optional fields: item_type: "Notebook" item_name: ["Hello World", "Hello World Subfolder"] file_path: - "/Hello World.Notebook/notebook-content.py" - "/subfolder/Hello World Subfolder.Notebook/notebook-content.py" # Lakehouse Connection Guid regex - find_value: \#\s*META\s+"default_lakehouse":\s*"([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})" replace_value: # Variable: $items.type.name.attribute (Note: item type and name values are CASE SENSITIVE; id attribute returns the deployed item's id/guid) PPE: "$items.Lakehouse.WithoutSchema.id" PROD: "$items.Lakehouse.WithoutSchema.id" # Optional fields: is_regex: "true" file_path: "/Example Notebook.Notebook/notebook-content.py" # Lakehouse workspace id regex - find_value: \#\s*META\s+"default_lakehouse_workspace_id":\s*"([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})" replace_value: # Variable: $workspace.id -> target workspace id PPE: "$workspace.id" PROD: "$workspace.id" # Optional fields: is_regex: "true" file_path: "/Example Notebook.Notebook/notebook-content.py" # SQL connection string - find_value: "sqlserverconnectionstringinoriginlakehouse.com" replace_value: # Variable: $items.type.name.attribute (Note: item type and name values are CASE SENSITIVE; sqlendpoint attribute returns the deployed item's sql endpoint) PPE: "$items.Lakehouse.WithoutSchema.sqlendpoint" PROD: "$items.Lakehouse.WithoutSchema.sqlendpoint" # Optional fields: file_path: "/Example Notebook.Notebook/notebook-content.py" # Eventhouse query URI - find_value: "https://trd-origineventhouse.z4.kusto.fabric.microsoft.com" replace_value: # Variable: $items.type.name.attribute (Note: item type and name values are CASE SENSITIVE; queryserviceuri attribute returns the deployed item's query service URI) PPE: "$items.Eventhouse.SampleEventhouse.queryserviceuri" PROD: "$items.Eventhouse.SampleEventhouse.queryserviceuri" # Optional fields: file_path: "/Example Notebook.Notebook/notebook-content.py" # Report byConnection - workspace id in connection string - find_value: "dev-workspace-id" replace_value: PPE: "$workspace.$id" PROD: "$workspace.$id" # Optional fields: file_path: "/ByConnection.Report/definition.pbir" # Report byConnection - semantic model name in connection string - find_value: "dev-semantic-model" replace_value: PPE: "ABC" PROD: "ABC" # Optional fields: file_path: "/ByConnection.Report/definition.pbir" # Report byConnection - semantic model id in connection string using dynamic replacement - find_value: "00000000-0000-0000-0000-000000000000" replace_value: # Variable: $items.type.name.attribute (Note: item type and name values are CASE SENSITIVE; id attribute returns the deployed item's id/guid) PPE: "$items.SemanticModel.ABC.$id" PROD: "$items.SemanticModel.ABC.$id" # Optional fields: file_path: "/ByConnection.Report/definition.pbir" key_value_replace: - find_key: $.variables[?(@.name=="SQL_Server")].value replace_value: PPE: "contoso-ppe.database.windows.net" PROD: "contoso-prod.database.windows.net" UAT: "contoso-uat.database.windows.net" # Optional fields: item_type: "VariableLibrary" item_name: "Vars" - find_key: $.variables[?(@.name=="Environment")].value replace_value: PPE: "PPE" PROD: "PROD" UAT: "UAT" # Optional fields: item_type: "VariableLibrary" item_name: "Vars" - find_key: $.variableOverrides[?(@.name=="SQL_Server")].value replace_value: PROD: "contoso-production-override.database.windows.net" file_path: Vars.VariableLibrary/valueSets/PROD.json item_type: "VariableLibrary" item_name: "Vars" - find_key: $.variableOverrides[?(@.name=="Environment")].value replace_value: PROD: "PROD_ENV" file_path: Vars.VariableLibrary/valueSets/PROD.json item_type: "VariableLibrary" item_name: "Vars" - find_key: $.schedules[?(@.jobType=="Execute")].enabled replace_value: PPE: false PROD: true file_path: "**/.schedules" spark_pool: # CapacityPool_Large - instance_pool_id: "72c68dbc-0775-4d59-909d-a47896f4573b" replace_value: PPE: type: "Capacity" name: "CapacityPool_Large_PPE" PROD: type: "Capacity" name: "CapacityPool_Large_PROD" # Optional field: item_name: "World" # CapacityPool_Medium - instance_pool_id: "e7b8f1c4-4a6e-4b8b-9b2e-8f1e5d6a9c3d" replace_value: PPE: type: "Workspace" name: "WorkspacePool_Medium" PROD: type: "Workspace" name: "WorkspacePool_Medium" # Optional field: item_name: # Supports default connections and per-item overrides (single connection only per model) semantic_model_binding: default: connection_id: PPE: "76e05dfe-9855-4e3d-a410-1dda048dbe99" PROD: "c4f8e2b1-3d2a-4f5b-9c6e-7a8b9c0d1e2f" models: - semantic_model_name: ["cloudconnections", "MySemanticModel_ADLS_Gen2"] connection_id: PPE: "f96870d5-5f86-49ad-bf41-5967fd7c1c6d" PROD: "a1b2c3d4-5678-90ab-cdef-1234567890ab" ================================================ FILE: sample/workspace/sample apache airflow job.ApacheAirflowJob/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "ApacheAirflowJob", "displayName": "sample apache airflow job" }, "config": { "version": "2.0", "logicalId": "f31cd938-c3ab-b977-4dd4-3e4a56d7c124" } } ================================================ FILE: sample/workspace/sample apache airflow job.ApacheAirflowJob/apacheairflowjob-content.json ================================================ { "properties": { "type": "Airflow", "typeProperties": { "airflowProperties": { "airflowConfigurationOverrides": {}, "airflowEnvironment": "FabricAirflowJob-1.0.0", "airflowRequirements": [], "airflowVersion": "2.10.5", "enableAADIntegration": true, "enableTriggerers": false, "environmentVariables": {}, "packageProviderPath": "plugins", "pythonVersion": "3.12" }, "computeProperties": { "computePool": "StarterPool", "computeSize": "Small", "enableAutoscale": false, "enableAvailabilityZones": false, "extraNodes": 0 } } } } ================================================ FILE: sample/workspace/sample apache airflow job.ApacheAirflowJob/dags/dag1.py ================================================ from datetime import datetime from airflow import DAG from airflow.operators.bash import BashOperator # Define the default arguments for the DAG default_args = { 'owner': 'airflow', 'depends_on_past': False, 'start_date': datetime(2023, 5, 1), 'email_on_failure': False, 'email_on_retry': False, 'retries': 1 } # Instantiate the DAG object with DAG( 'dags-dag1.py', default_args=default_args, description='A simple Hello World DAG', schedule_interval=None, catchup=False ) as dag: # Define the tasks hello_task = BashOperator( task_id='hello_world_task', bash_command='echo "Hello, World!"' ) # Set the task dependencies hello_task ================================================ FILE: sample/workspace/subfolder/Hello World Subfolder.Notebook/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "Notebook", "displayName": "Hello World Subfolder", "description": "New notebook" }, "config": { "version": "2.0", "logicalId": "2d147568-6f37-9881-461d-c07a95bcb35d" } } ================================================ FILE: sample/workspace/subfolder/Hello World Subfolder.Notebook/notebook-content.py ================================================ # Fabric notebook source # METADATA ******************** # META { # META "kernel_info": { # META "name": "synapse_pyspark" # META }, # META "dependencies": { # META "environment": { # META "environmentId": "a277ea4a-e87f-8537-4ce0-39db11d4aade", # META "workspaceId": "00000000-0000-0000-0000-000000000000" # META } # META } # META } # CELL ******************** print("Hello World") # METADATA ******************** # META { # META "language": "python", # META "language_group": "synapse_pyspark" # META } ================================================ FILE: sample/workspace/subfolder/subfolder/Hello World SubfolderSubfolder.Notebook/.platform ================================================ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "Notebook", "displayName": "Hello World SubfolderSubfolder", "description": "New notebook" }, "config": { "version": "2.0", "logicalId": "66dd4a2e-45b9-aff1-418c-695af0f1885a" } } ================================================ FILE: sample/workspace/subfolder/subfolder/Hello World SubfolderSubfolder.Notebook/notebook-content.py ================================================ # Fabric notebook source # METADATA ******************** # META { # META "kernel_info": { # META "name": "synapse_pyspark" # META }, # META "dependencies": { # META "environment": { # META "environmentId": "a277ea4a-e87f-8537-4ce0-39db11d4aade", # META "workspaceId": "00000000-0000-0000-0000-000000000000" # META } # META } # META } # CELL ******************** print("Hello World") # METADATA ******************** # META { # META "language": "python", # META "language_group": "synapse_pyspark" # META } ================================================ FILE: sample/workspace/templates/nb parameter template 1.yml ================================================ find_replace: # SQL connection string - find_value: "sqlserverconnectionstringinoriginlakehouse.com" replace_value: # Variable: $items.type.name.attribute (Note: item type and name values are CASE SENSITIVE; sqlendpoint attribute returns the deployed item's sql endpoint) PPE: "$items.Lakehouse.WithoutSchema.sqlendpoint" PROD: "$items.Lakehouse.WithoutSchema.sqlendpoint" # Optional fields: file_path: "/Example Notebook.Notebook/notebook-content.py" # Eventhouse query URI - find_value: "https://trd-origineventhouse.z4.kusto.fabric.microsoft.com" replace_value: # Variable: $items.type.name.attribute (Note: item type and name values are CASE SENSITIVE; queryserviceuri attribute returns the deployed item's query service URI) PPE: "$items.Eventhouse.SampleEventhouse.queryserviceuri" PROD: "$items.Eventhouse.SampleEventhouse.queryserviceuri" # Optional fields: file_path: "/Example Notebook.Notebook/notebook-content.py" ================================================ FILE: sample/workspace/templates/nb parameter template 2.yml ================================================ find_replace: # Lakehouse Connection Guid regex - find_value: \#\s*META\s+"default_lakehouse":\s*"([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})" replace_value: # Variable: $items.type.name.attribute (Note: item type and name values are CASE SENSITIVE; id attribute returns the deployed item's id/guid) PPE: "$items.Lakehouse.WithoutSchema.id" PROD: "$items.Lakehouse.WithoutSchema.id" # Optional fields: is_regex: "true" file_path: "/Example Notebook.Notebook/notebook-content.py" # Lakehouse workspace id regex - find_value: \#\s*META\s+"default_lakehouse_workspace_id":\s*"([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})" replace_value: # Variable: $workspace.id -> target workspace id PPE: "$workspace.id" PROD: "$workspace.id" # Optional fields: is_regex: "true" file_path: "/Example Notebook.Notebook/notebook-content.py" ================================================ FILE: src/fabric_cicd/__init__.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Provides tools for managing and publishing items in a Fabric workspace.""" import logging import sys import fabric_cicd.constants as constants from fabric_cicd._common._deployment_result import DeploymentResult, DeploymentStatus from fabric_cicd._common._git_diff_utils import get_changed_items from fabric_cicd._common._logging import configure_logger, exception_handler, get_file_handler from fabric_cicd._common._validate_env_vars import _get_fabric_fqdn_url, validate_api_url from fabric_cicd._common._validate_input import validate_workspace_id from fabric_cicd.constants import FeatureFlag, ItemType from fabric_cicd.fabric_workspace import FabricWorkspace from fabric_cicd.publish import deploy_with_config, publish_all_items, unpublish_all_orphan_items logger = logging.getLogger(__name__) def append_feature_flag(feature: str) -> None: """ Append a feature flag to the global feature_flag set. Args: feature: The feature flag to be included. Examples: Basic usage >>> from fabric_cicd import append_feature_flag >>> append_feature_flag("enable_lakehouse_unpublish") """ constants.FEATURE_FLAG.add(feature) def change_log_level(level: str = "DEBUG") -> None: """ Sets the log level for all loggers within the fabric_cicd package. Currently only supports DEBUG. Args: level: The logging level to set (e.g., DEBUG). Examples: Basic usage >>> from fabric_cicd import change_log_level >>> change_log_level("DEBUG") """ if level.upper() == "DEBUG": configure_logger(logging.DEBUG) logger.info("Changed log level to DEBUG") else: logger.warning(f"Log level '{level}' not supported. Only DEBUG is supported at this time. No changes made.") def configure_external_file_logging(external_logger: logging.Logger) -> None: """ Configure fabric_cicd package logging to integrate with an external logger's file handler. This is an advanced alternative to the default file logging configuration when level is set to DEBUG via `change_log_level()`. Extracts the file handler from the provided logger and configures fabric_cicd to append only DEBUG logs (e.g., API request/response details) to the same file. The external logger retains full ownership of the handler, including file rotation (if applicable) and lifecycle management. Note: - This function resets logging configuration. Use as an alternative to ``change_log_level()`` or ``disable_file_logging()``, not in combination - Only DEBUG logs from the fabric_cicd package are written to the log file. Exception messages are displayed on the console, but full stack traces are not written to the external log file - Console output remains at INFO level (default fabric_cicd console behavior) Args: external_logger: The external logger instance that has a `FileHandler` or `RotatingFileHandler` attached. Raises: ValueError: If no file handler is found on the provided logger. Examples: General usage: >>> import logging >>> from logging.handlers import RotatingFileHandler >>> from fabric_cicd import configure_external_file_logging ... >>> # Set up your own logger with a file handler >>> my_logger = logging.getLogger("MyApp") >>> handler = RotatingFileHandler("app.log", maxBytes=5*1024*1024, backupCount=7) >>> handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")) >>> my_logger.addHandler(handler) ... >>> # Configure fabric_cicd to use the same file >>> configure_external_file_logging(my_logger) """ # Extract file handler from external logger file_handler = get_file_handler(external_logger) if file_handler is None: msg = "No FileHandler or RotatingFileHandler found on the provided logger." raise ValueError(msg) configure_logger( level=logging.DEBUG, suppress_debug_console=True, debug_only_file=True, external_file_handler=file_handler, ) def disable_file_logging() -> None: """ Disable file logging for the fabric_cicd package. When called, no log file will be created and only console logging will occur at the default INFO level. Note: - This function is intended to be used as an alternative to `change_log_level()` or `configure_external_file_logging()`, not in combination with them as this will reset logging configurations to INFO-level console output only. - Exception messages will still be displayed on the console, but full stack traces will not be written to any log file or console. Examples: Basic usage >>> from fabric_cicd import disable_file_logging >>> disable_file_logging() """ configure_logger(disable_log_file=True) def configure_fabric_fqdn(workspace_id: str) -> None: """ Configure Fabric API URLs for private-link-enabled workspaces. Updates the global Fabric API URL constants to use the FQDN format required for private-link-enabled workspaces. Call this function before initializing a FabricWorkspace if you are using a private-link-enabled workspace. Args: workspace_id: The workspace ID string in standard GUID format with dashes (e.g., "f953f3da-c5f0-4e36-a644-c85933e35e2f"). Side Effects: Updates the module-level constants in fabric_cicd.constants: - FABRIC_API_ROOT_URL: Set to the FQDN URL derived from workspace_id - DEFAULT_API_ROOT_URL: Set to the same FQDN URL Examples: Basic usage with FabricWorkspace: >>> from fabric_cicd import configure_fabric_fqdn, FabricWorkspace >>> from azure.identity import AzureCliCredential >>> >>> workspace_id = "f953f3da-c5f0-4e36-a644-c85933e35e2f" >>> configure_fabric_fqdn(workspace_id) >>> >>> token_credential = AzureCliCredential() >>> workspace = FabricWorkspace( ... workspace_id=workspace_id, ... repository_directory="/path/to/workspace", ... token_credential=token_credential ... ) """ workspace_id = validate_workspace_id(workspace_id) fqdn_url = _get_fabric_fqdn_url(workspace_id) fqdn_url = validate_api_url(fqdn_url, "configure_fabric_fqdn") if constants.FABRIC_API_ROOT_URL != "https://api.fabric.microsoft.com": logger.warning( f"configure_fabric_fqdn: overwriting previously set FABRIC_API_ROOT_URL '{constants.FABRIC_API_ROOT_URL}'" ) constants.FABRIC_API_ROOT_URL = fqdn_url constants.DEFAULT_API_ROOT_URL = fqdn_url configure_logger() sys.excepthook = exception_handler __all__ = [ "DeploymentResult", "DeploymentStatus", "FabricWorkspace", "FeatureFlag", "ItemType", "append_feature_flag", "change_log_level", "configure_external_file_logging", "configure_fabric_fqdn", "deploy_with_config", "disable_file_logging", "get_changed_items", "publish_all_items", "unpublish_all_orphan_items", ] ================================================ FILE: src/fabric_cicd/_common/__init__.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. ================================================ FILE: src/fabric_cicd/_common/_check_utils.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Utility functions for checking file types and versions.""" import json import logging import re from pathlib import Path import filetype import yaml from fabric_cicd._common._exceptions import FileTypeError logger = logging.getLogger(__name__) def check_file_type(file_path: Path) -> str: """ Check the type of the provided file. Args: file_path: The path to the file. """ try: kind = filetype.guess(file_path) except Exception as e: msg = f"Error determining file type of {file_path}: {e}" FileTypeError(msg, logger) if kind is not None: if kind.mime.startswith("application/"): return "binary" if kind.mime.startswith("image/"): return "image" return "text" def check_regex(regex: str) -> re.Pattern: """ Check if a regex pattern is valid and returns the pattern. Args: regex: The regex pattern to match. """ try: regex_pattern = re.compile(regex) except Exception as e: msg = f"An error occurred with the regex provided: {e}" raise ValueError(msg) from e return regex_pattern def check_valid_json_content(content: str) -> bool: """ Check if the given string content is valid JSON. Args: content: The string content to validate as JSON. Returns: bool: True if the content is valid JSON, False otherwise. """ try: json.loads(content) return True except json.JSONDecodeError: return False def check_valid_yaml_content(content: str) -> bool: """ Check if the given string content is valid structured YAML (mapping or sequence). Args: content: The string content to validate as YAML. Returns: bool: True if the content parses as a YAML mapping or sequence, False otherwise. """ try: result = yaml.safe_load(content) return isinstance(result, (dict, list)) except yaml.YAMLError: return False ================================================ FILE: src/fabric_cicd/_common/_color.py ================================================ class Fore: BLACK = "\033[30m" RED = "\033[31m" GREEN = "\033[32m" YELLOW = "\033[33m" BLUE = "\033[34m" MAGENTA = "\033[35m" CYAN = "\033[36m" WHITE = "\033[37m" RESET = "\033[39m" class Back: BLACK = "\033[40m" RED = "\033[41m" GREEN = "\033[42m" YELLOW = "\033[43m" BLUE = "\033[44m" MAGENTA = "\033[45m" CYAN = "\033[46m" WHITE = "\033[47m" RESET = "\033[49m" class Style: BRIGHT = "\033[1m" DIM = "\033[2m" NORMAL = "\033[22m" RESET_ALL = "\033[0m" ================================================ FILE: src/fabric_cicd/_common/_config_utils.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Utilities for YAML-based deployment configuration.""" import contextlib import logging from collections.abc import Generator from typing import Optional, Union from fabric_cicd import constants from fabric_cicd._common._config_validator import ConfigValidator logger = logging.getLogger(__name__) def load_config_file(config_file_path: str, environment: str, config_override: Optional[dict] = None) -> dict: """Load and validate YAML configuration file. Args: config_file_path: Path to the YAML config file environment: Target environment for deployment config_override: Optional dictionary to override specific configuration values Returns: Parsed and validated configuration dictionary """ validator = ConfigValidator() return validator.validate_config_file(config_file_path, environment, config_override) def get_config_value(config_section: dict, key: str, environment: str) -> Optional[Union[str, list, bool]]: """Extract a value from config, handling both single and environment-specific formats. Args: config_section: The config section to extract from key: The key to extract environment: Target environment Returns: The extracted value, or None if key doesn't exist or environment not found in dict """ if key not in config_section: return None value = config_section[key] if isinstance(value, dict): return value.get(environment) return value def update_setting( settings: dict, config: dict, key: str, environment: str, default_value: Optional[str] = None, output_key: Optional[str] = None, ) -> None: """ Gets a config value using get_config_value and updates the settings dictionary if the value is not None. Args: settings: The settings dictionary to update config: The configuration dictionary key: The key to extract from the config environment: Target environment default_value: The default value to set if the config value is None output_key: The key to use in the settings dictionary (defaults to `key` if None) """ value = get_config_value(config, key, environment) target_key = output_key or key if value is not None: settings[target_key] = value elif default_value is not None: settings[target_key] = default_value def extract_workspace_settings(config: dict, environment: str) -> dict: """Extract workspace-specific settings from config for the given environment.""" environment = environment.strip() core = config["core"] settings = {} # Workspace ID or name - required, validation ensures value exists for target environment if "workspace_id" in core: settings["workspace_id"] = get_config_value(core, "workspace_id", environment) logger.info(f"Using workspace ID '{settings['workspace_id']}'") elif "workspace" in core: settings["workspace_name"] = get_config_value(core, "workspace", environment) logger.info(f"Using workspace '{settings['workspace_name']}'") # Repository directory - required, validation ensures value exists for target environment if "repository_directory" in core: settings["repository_directory"] = get_config_value(core, "repository_directory", environment) # Optional settings - validation logs warning if value not found for target environment update_setting(settings, core, "item_types_in_scope", environment) update_setting(settings, core, "parameter", environment, output_key="parameter_file_path") return settings def extract_publish_settings(config: dict, environment: str) -> dict: """Extract publish-specific settings from config for the given environment.""" settings = {} if "publish" in config: publish_config = config["publish"] # Optional settings - validation logs debug if value not found for target environment settings_to_update = [ "exclude_regex", "folder_exclude_regex", "folder_path_to_include", "items_to_include", "shortcut_exclude_regex", ] for key in settings_to_update: update_setting(settings, publish_config, key, environment) # Skip defaults to False if setting not found update_setting(settings, publish_config, "skip", environment, default_value=False) return settings def extract_unpublish_settings(config: dict, environment: str) -> dict: """Extract unpublish-specific settings from config for the given environment.""" settings = {} if "unpublish" in config: unpublish_config = config["unpublish"] # Optional settings - validation logs debug if value not found for target environment settings_to_update = [ "exclude_regex", "items_to_include", ] for key in settings_to_update: update_setting(settings, unpublish_config, key, environment) # Skip defaults to False if setting not found update_setting(settings, unpublish_config, "skip", environment, default_value=False) return settings @contextlib.contextmanager def config_overrides_scope(config: dict, environment: str) -> Generator[None, None, None]: """ Context manager that applies feature flags and constants overrides from config and guarantees cleanup. Feature flags and constants are restored to their pre-call values on exit, ensuring no state leaks between deployments. Args: config: Configuration dictionary environment: Target environment for deployment """ # Snapshot current state before any changes original_feature_flags = constants.FEATURE_FLAG.copy() overridden_keys = {} try: # Set feature flags if "features" in config: features = config["features"] features_list = features.get(environment, []) if isinstance(features, dict) else features for feature in features_list: constants.FEATURE_FLAG.add(feature) logger.info(f"Enabled feature flag: {feature}") # Apply constants overrides if "constants" in config: constants_section = config["constants"] for key in list(constants_section.keys()): value = get_config_value(constants_section, key, environment) if value is not None and hasattr(constants, key): overridden_keys[key] = getattr(constants, key) setattr(constants, key, value) logger.warning(f"Override constant {key} = {value}") yield finally: # Restore original state — guaranteed even if deployment raises constants.FEATURE_FLAG.clear() constants.FEATURE_FLAG.update(original_feature_flags) for key, original_value in overridden_keys.items(): setattr(constants, key, original_value) ================================================ FILE: src/fabric_cicd/_common/_config_validator.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Configuration validation for YAML-based deployment configuration.""" import logging import re from pathlib import Path from typing import Any, Optional, Union import yaml from fabric_cicd import constants from fabric_cicd._common._exceptions import InputError from fabric_cicd._common._validate_env_vars import _URL_CONSTANTS, validate_api_url logger = logging.getLogger(__name__) class ConfigValidationError(InputError): """Specific exception for configuration validation errors.""" def __init__(self, errors: list[str], logger_instance: logging.Logger) -> None: """Initialize with list of validation errors.""" self.validation_errors = errors error_msg = f"Configuration validation failed with {len(errors)} error(s):\n" + "\n".join( f" - {error}" for error in errors ) super().__init__(error_msg, logger_instance) class ConfigValidator: """Validates YAML configuration files for fabric-cicd deployment.""" def __init__(self) -> None: """Initialize the validator.""" self.errors: list = [] self.config: dict = None self.config_path: Path = None self.environment: str = None self.config_override: Optional[dict] = None def validate_config_file( self, config_file_path: str, environment: str, config_override: Optional[dict] = None ) -> dict[str, Any]: """ Validate configuration file and return parsed config if valid. Args: config_file_path: String path to the configuration file environment: The target environment for the deployment config_override: Optional dictionary to override specific configuration values Returns: Parsed configuration dictionary (includes overrides, if any) Raises: ConfigValidationError: If validation fails """ self.errors = [] self.environment = environment self.config_override = config_override # Step 1: Validate file existence and accessibility config_path = self._validate_file_existence(config_file_path) # Step 2: Validate file content and YAML syntax self.config = self._validate_yaml_content(config_path) # Step 3: Apply and validate config overrides if self.config is not None and self.config_override is not None: self._apply_and_validate_overrides() # Step 4: Validate configuration structure and required fields if self.config is not None: self._validate_config_structure() self._validate_config_sections() # Step 5: Validate environment-specific mapping self._validate_environment_exists() # Step 6: Resolve paths after environment validation passes if not self.errors: self._resolve_repository_path() self._resolve_parameter_path() # If there are validation errors, raise them all at once if self.errors: raise ConfigValidationError(self.errors, logger) return self.config def _validate_file_existence(self, config_file_path: str) -> Path: """Validate file path and existence.""" if not config_file_path or not isinstance(config_file_path, str): self.errors.append(constants.CONFIG_VALIDATION_MSGS["file"]["path_empty"]) return None try: config_path = Path(config_file_path).resolve() except (OSError, RuntimeError) as e: self.errors.append(constants.CONFIG_VALIDATION_MSGS["file"]["invalid_path"].format(config_file_path, e)) return None if not config_path.exists(): self.errors.append(constants.CONFIG_VALIDATION_MSGS["file"]["not_found"].format(config_file_path)) return None if not config_path.is_file(): self.errors.append(constants.CONFIG_VALIDATION_MSGS["file"]["not_file"].format(config_file_path)) return None self.config_path = config_path return config_path def _validate_yaml_content(self, config_path: Optional[Path]) -> Optional[dict]: """Validate YAML syntax and basic structure.""" if config_path is None: return None try: with config_path.open(encoding="utf-8") as f: config = yaml.safe_load(f) except yaml.YAMLError as e: self.errors.append(constants.CONFIG_VALIDATION_MSGS["file"]["yaml_syntax"].format(e)) return None except UnicodeDecodeError as e: self.errors.append(constants.CONFIG_VALIDATION_MSGS["file"]["encoding_error"].format(e)) return None except PermissionError as e: self.errors.append(constants.CONFIG_VALIDATION_MSGS["file"]["permission_denied"].format(e)) return None except Exception as e: self.errors.append(constants.CONFIG_VALIDATION_MSGS["file"]["unexpected_error"].format(e)) return None # Handle empty file case if config is None: self.errors.append(constants.CONFIG_VALIDATION_MSGS["file"]["empty_file"]) return None if not isinstance(config, dict): self.errors.append(constants.CONFIG_VALIDATION_MSGS["file"]["not_dict"].format(type(config).__name__)) return None return config def _apply_and_validate_overrides(self) -> None: """Apply and validate config overrides.""" if not self.config_override: return for section, value in self.config_override.items(): try: # Validate the section if not self._valid_override_section(section, value): continue # Merge overrides into config self._merge_overrides(section, value) except Exception as e: self.errors.append(constants.CONFIG_VALIDATION_MSGS["override"]["apply_failed"].format(section, e)) def _valid_override_section(self, section: str, value: any) -> bool: """Validates the override section and structure are correct.""" # Check section is supported if section not in constants.CONFIG_SECTIONS: self.errors.append( constants.CONFIG_VALIDATION_MSGS["override"]["unsupported_section"].format( section, list(constants.CONFIG_SECTIONS.keys()) ) ) return False # Check type is valid expected_types = constants.CONFIG_SECTIONS[section]["type"] if not isinstance(value, expected_types): type_names = ( " or ".join(t.__name__ for t in expected_types) if isinstance(expected_types, tuple) else expected_types.__name__ ) self.errors.append( constants.CONFIG_VALIDATION_MSGS["override"]["wrong_type"].format( section, type_names, type(value).__name__ ) ) return False # Check setting is supported for applicable sections if isinstance(value, dict) and section in ["core", "publish", "unpublish"]: supported = constants.CONFIG_SECTIONS[section]["settings"] for setting in value: if setting not in supported: self.errors.append( constants.CONFIG_VALIDATION_MSGS["override"]["unsupported_setting"].format( section, setting, supported ) ) return False return True def _merge_overrides(self, section: str, value: Union[dict, list]) -> None: """Merge section and setting overrides into config file.""" # Special handling for features and constants sections if section == "features": action = "added" if not self.config.get("features") else "updated" self.config["features"] = value logger.warning(constants.CONFIG_VALIDATION_MSGS["log"]["override_section"].format(action, section, value)) return if section == "constants": action = "added" if "constants" not in self.config else "updated" self.config["constants"] = value logger.warning(constants.CONFIG_VALIDATION_MSGS["log"]["override_section"].format(action, section, value)) return # Add section if it doesn't already exist (publish, unpublish only) if section not in self.config: if section == "core": self.errors.append(constants.CONFIG_VALIDATION_MSGS["override"]["cannot_create_core"]) return self.config[section] = {} logger.warning(constants.CONFIG_VALIDATION_MSGS["log"]["override_added_section"].format(section)) # Process field by field for other sections (core, publish, unpublish) for setting, setting_value in value.items(): exists = setting in self.config[section] # Validate required fields can only be overridden, not added if not exists and section == "core": if setting == "repository_directory": self.errors.append( constants.CONFIG_VALIDATION_MSGS["override"]["cannot_create_required"].format(setting) ) continue if setting in ["workspace_id", "workspace"]: # Check if the other workspace identifier exists other_workspace_field = "workspace" if setting == "workspace_id" else "workspace_id" if other_workspace_field not in self.config[section]: self.errors.append( constants.CONFIG_VALIDATION_MSGS["override"]["cannot_create_workspace_id"].format(setting) ) continue # Handle environment specific override if isinstance(setting_value, dict) and self.environment in setting_value: env_value = setting_value[self.environment] # Replace existing environment value with override value if exists and isinstance(self.config[section][setting], dict): self.config[section][setting][self.environment] = env_value logger.warning( constants.CONFIG_VALIDATION_MSGS["log"]["override_env_specific"].format( section, setting, self.environment, env_value ) ) # Otherwise, add new environment value else: self.config[section][setting] = {self.environment: env_value} logger.warning( constants.CONFIG_VALIDATION_MSGS["log"]["override_env_mapping"].format( section, setting, self.environment, env_value ) ) # Otherwise, handle direct value override else: self.config[section][setting] = setting_value action = "updated" if exists else "added" logger.warning( constants.CONFIG_VALIDATION_MSGS["log"]["override_setting"].format( action, section, setting, setting_value ) ) def _validate_config_structure(self) -> None: """Validate top-level configuration structure.""" if not isinstance(self.config, dict): return # Check for required top-level sections if "core" not in self.config: self.errors.append(constants.CONFIG_VALIDATION_MSGS["structure"]["missing_core"]) return if not isinstance(self.config["core"], dict): self.errors.append( constants.CONFIG_VALIDATION_MSGS["structure"]["core_not_dict"].format( type(self.config["core"]).__name__ ) ) def _validate_config_sections(self) -> None: """Validate the configuration sections""" # Validate core section (required) if "core" not in self.config or not isinstance(self.config["core"], dict): self.errors.append(constants.CONFIG_VALIDATION_MSGS["structure"]["missing_core"]) return core = self.config["core"] # Validate workspace identification (must have either workspace_id or workspace) has_workspace_id = self._validate_workspace_field(core, "workspace_id") has_workspace_name = self._validate_workspace_field(core, "workspace") if not has_workspace_id and not has_workspace_name: self.errors.append(constants.CONFIG_VALIDATION_MSGS["structure"]["missing_workspace_id"]) # Validate repository_directory self._validate_repository_directory(core) # Validate optional item_types_in_scope self._validate_item_types_in_scope(core) # Validate optional parameter field self._validate_parameter_field(core) # Validate optional sections # publish section if "publish" in self.config: self._validate_operation_section(self.config["publish"], "publish") # unpublish section if "unpublish" in self.config: self._validate_operation_section(self.config["unpublish"], "unpublish") # features section if "features" in self.config: self._validate_features_section(self.config["features"]) # constants section if "constants" in self.config: self._validate_constants_section(self.config["constants"]) def _validate_environment_exists(self) -> None: """Validate that target environment exists in all environment mappings.""" if self.environment == "N/A": # Handle no target environment case if any( field_name in section and isinstance(section[field_name], dict) for section, field_name, _, _, _ in _get_config_fields(self.config) if field_name != "constants" ): self.errors.append(constants.CONFIG_VALIDATION_MSGS["environment"]["no_env_with_mappings"]) return # Check each field for target environment presence for section, field_name, display_name, is_required, log_warning in _get_config_fields(self.config): if field_name in section: field_value = section[field_name] # Handle constants special case — check each constant's value individually if field_name == "constants": if isinstance(field_value, dict): for const_key, const_val in field_value.items(): if isinstance(const_val, dict) and self.environment not in const_val: available_envs = list(const_val.keys()) logger.warning( f"Environment '{self.environment}' not found in 'constants.{const_key}'. " f"Available environments: {available_envs}. This constant will be skipped." ) continue # If it's a dict (environment mapping), check if target environment exists if isinstance(field_value, dict) and self.environment not in field_value: available_envs = list(field_value.keys()) msg = ( f"Environment '{self.environment}' not found in '{display_name}'. " f"Available environments: {available_envs}. This setting will be skipped." ) if is_required: self.errors.append( constants.CONFIG_VALIDATION_MSGS["environment"]["env_not_found"].format( self.environment, display_name, available_envs ) ) elif log_warning: logger.warning(msg) else: logger.debug(msg) def _validate_environment_mapping(self, field_value: dict, field_name: str, accepted_type: type) -> bool: """Validate field with environment mapping.""" if not field_value: self.errors.append(constants.CONFIG_VALIDATION_MSGS["environment"]["empty_mapping"].format(field_name)) return False valid = True for env, value in field_value.items(): # Validate environment key if not isinstance(env, str) or not env.strip(): self.errors.append( constants.CONFIG_VALIDATION_MSGS["environment"]["invalid_env_key"].format( field_name, type(env).__name__ ) ) valid = False continue # Validate environment value type if not isinstance(value, accepted_type): self.errors.append( f"'{field_name}' value for environment '{env}' must be a {accepted_type.__name__}, got {type(value).__name__}" ) valid = False continue # Validate environment value content (type-specific) if accepted_type == str: if not value.strip(): self.errors.append( constants.CONFIG_VALIDATION_MSGS["environment"]["empty_env_value"].format(field_name, env) ) valid = False elif accepted_type == list and not value: self.errors.append( constants.CONFIG_VALIDATION_MSGS["environment"]["empty_env_value"].format(field_name, env) ) valid = False return valid def _validate_workspace_field(self, core: dict, field_name: str) -> bool: """Validate workspace_id or workspace field.""" if field_name not in core: return False field_value = core[field_name] # Support both string values and environment mappings if isinstance(field_value, str): if not field_value.strip(): self.errors.append(constants.CONFIG_VALIDATION_MSGS["field"]["empty_value"].format(field_name)) return False return self._validate_workspace_value(field_value, field_name, field_name) if isinstance(field_value, dict): valid = self._validate_environment_mapping(field_value, field_name, str) # Apply field-specific validation to each environment value if valid: for env, value in field_value.items(): if isinstance(value, str) and not self._validate_workspace_value( value, field_name, f"{field_name}.{env}" ): valid = False return valid self.errors.append( constants.CONFIG_VALIDATION_MSGS["field"]["string_or_dict"].format(field_name, type(field_value).__name__) ) return False def _validate_workspace_value(self, value: str, field_name: str, context: str) -> bool: """Validate a workspace value (applies GUID validation for workspace_id).""" if field_name == "workspace_id" and not _validate_guid_format(value): self.errors.append(constants.CONFIG_VALIDATION_MSGS["field"]["invalid_guid"].format(context, value)) return False return True def _validate_repository_directory(self, core: dict) -> None: """Validate repository_directory field.""" if "repository_directory" not in core: self.errors.append(constants.CONFIG_VALIDATION_MSGS["structure"]["missing_repository_dir"]) return repository_directory = core["repository_directory"] # Support both string values and environment mappings if isinstance(repository_directory, str): if not repository_directory.strip(): self.errors.append( constants.CONFIG_VALIDATION_MSGS["field"]["empty_value"].format("repository_directory") ) return elif isinstance(repository_directory, dict): if not self._validate_environment_mapping(repository_directory, "repository_directory", str): return else: self.errors.append( constants.CONFIG_VALIDATION_MSGS["field"]["string_or_dict"].format( "repository_directory", type(repository_directory).__name__ ) ) return def _validate_item_types_in_scope(self, core: dict[str, Any]) -> None: """Validate item_types_in_scope field if present.""" if "item_types_in_scope" not in core: return # Optional field item_types = core["item_types_in_scope"] if isinstance(item_types, list): if not item_types: self.errors.append( constants.CONFIG_VALIDATION_MSGS["field"]["empty_list"].format("item_types_in_scope") ) return self._validate_item_types(item_types) return if isinstance(item_types, dict): # Validate environment mapping if not self._validate_environment_mapping(item_types, "item_types_in_scope", list): return # Validate each environment's item types for env, item_type_list in item_types.items(): self._validate_item_types(item_type_list, env_context=env) return self.errors.append( constants.CONFIG_VALIDATION_MSGS["field"]["item_types_list_or_dict"].format(type(item_types).__name__) ) def _validate_item_types(self, item_types: list, env_context: Optional[str] = None) -> None: """Validate a list of item types.""" if not item_types: self.errors.append(constants.CONFIG_VALIDATION_MSGS["field"]["empty_list"].format("item_types_in_scope")) return # Validate each item type for item_type in item_types: if not isinstance(item_type, str): self.errors.append( constants.CONFIG_VALIDATION_MSGS["field"]["invalid_item_type"].format( type(item_type).__name__, item_type ) ) continue if item_type not in constants.ACCEPTED_ITEM_TYPES: available_types = ", ".join(sorted(constants.ACCEPTED_ITEM_TYPES)) if env_context: self.errors.append( constants.CONFIG_VALIDATION_MSGS["field"]["unsupported_item_type_env"].format( item_type, env_context, available_types ) ) else: self.errors.append( constants.CONFIG_VALIDATION_MSGS["field"]["unsupported_item_type"].format( item_type, available_types ) ) def _validate_parameter_field(self, core: dict) -> None: """Validate parameter field if present.""" if "parameter" not in core: return # Optional field parameter_value = core["parameter"] # Support both string values and environment mappings if isinstance(parameter_value, str): if not parameter_value.strip(): self.errors.append(constants.CONFIG_VALIDATION_MSGS["field"]["empty_value"].format("parameter")) return elif isinstance(parameter_value, dict): if not self._validate_environment_mapping(parameter_value, "parameter", str): return else: self.errors.append( constants.CONFIG_VALIDATION_MSGS["field"]["string_or_dict"].format( "parameter", type(parameter_value).__name__ ) ) def _resolve_path_field( self, field_value: Union[str, dict], field_name: str, section_name: str, path_type: str = "directory" ) -> None: """Path resolution for configuration "path" fields (e.g, repository_directory, parameter).""" # Prepare paths for resolution paths_to_resolve = {"_default": field_value} if isinstance(field_value, str) else field_value # Skip resolution if config validation failed if not self.config_path: logger.debug(constants.CONFIG_VALIDATION_MSGS["path"]["skip"].format(field_name)) return # If environment mapping is used and target environment is provided, only process that environment path if self.environment and self.environment != "N/A" and isinstance(field_value, dict): if self.environment not in paths_to_resolve: # Skip if environment not in mapping (for parameter field, which is optional) logger.debug( f"Skipping path resolution for '{field_name}' - environment '{self.environment}' not in mapping" ) return paths_to_resolve = {self.environment: paths_to_resolve[self.environment]} for env_key, path_str in paths_to_resolve.items(): try: env_desc = f" for environment '{env_key}'" if env_key != "_default" else "" path = Path(path_str) if path.is_absolute(): resolved_path = path logger.info( constants.CONFIG_VALIDATION_MSGS["path"]["absolute"].format(field_name, env_desc, resolved_path) ) # Validate absolute paths are in the same git repository as config file config_repo_root = _find_git_root(self.config_path.parent) path_repo_root = _find_git_root(resolved_path.parent if path_type == "file" else resolved_path) if config_repo_root and path_repo_root and config_repo_root != path_repo_root: self.errors.append( constants.CONFIG_VALIDATION_MSGS["path"]["git_repo"].format( field_name, env_desc, config_repo_root, field_name, path_repo_root ) ) continue else: # Resolve relative to config path location config_dir = self.config_path.parent resolved_path = (config_dir / path_str).resolve() logger.info( constants.CONFIG_VALIDATION_MSGS["path"]["resolved"].format( field_name, path_str, env_desc, resolved_path ) ) # Validate the resolved path exists if not resolved_path.exists(): self.errors.append( constants.CONFIG_VALIDATION_MSGS["path"]["not_found"].format( field_name, env_desc, resolved_path ) ) continue # Path type-specific validation if path_type == "directory": if not resolved_path.is_dir(): self.errors.append( constants.CONFIG_VALIDATION_MSGS["path"]["not_directory"].format( field_name, env_desc, resolved_path ) ) continue elif path_type == "file" and not resolved_path.is_file(): self.errors.append( constants.CONFIG_VALIDATION_MSGS["path"]["not_file"].format(field_name, env_desc, resolved_path) ) continue # Store the resolved path back in config if isinstance(field_value, str): if section_name: self.config[section_name][field_name] = str(resolved_path) else: if section_name: self.config[section_name][field_name][env_key] = str(resolved_path) except (OSError, ValueError) as e: self.errors.append( constants.CONFIG_VALIDATION_MSGS["path"]["invalid"].format(field_name, path_str, env_desc, e) ) def _resolve_repository_path(self) -> None: """Resolve repository directory paths after environment validation.""" core = self.config["core"] repository_directory = core["repository_directory"] self._resolve_path_field(repository_directory, "repository_directory", "core", "directory") def _resolve_parameter_path(self) -> None: """Resolve parameter file paths after environment validation.""" core = self.config["core"] if "parameter" not in core: return # Optional field parameter_value = core["parameter"] self._resolve_path_field(parameter_value, "parameter", "core", "file") def _validate_operation_section(self, section: dict[str, Any], section_name: str) -> None: """Validate publish/unpublish section structure.""" if not isinstance(section, dict): self.errors.append( constants.CONFIG_VALIDATION_MSGS["operation"]["not_dict"].format(section_name, type(section).__name__) ) return # Validate exclude_regex if present if "exclude_regex" in section: exclude_regex = section["exclude_regex"] if isinstance(exclude_regex, str): if not exclude_regex.strip(): self.errors.append( constants.CONFIG_VALIDATION_MSGS["operation"]["empty_string"].format( f"{section_name}.exclude_regex" ) ) else: self._validate_regex(exclude_regex, f"{section_name}.exclude_regex") elif isinstance(exclude_regex, dict): # Validate environment mapping if not self._validate_environment_mapping(exclude_regex, f"{section_name}.exclude_regex", str): return # Validate each environment's regex pattern for env, regex_pattern in exclude_regex.items(): self._validate_regex(regex_pattern, f"{section_name}.exclude_regex.{env}") else: self.errors.append( constants.CONFIG_VALIDATION_MSGS["field"]["string_or_dict"].format( f"{section_name}.exclude_regex", type(exclude_regex).__name__ ) ) # Validate items_to_include if present if "items_to_include" in section: items = section["items_to_include"] if isinstance(items, list): if not items: self.errors.append( constants.CONFIG_VALIDATION_MSGS["operation"]["empty_list"].format( f"{section_name}.items_to_include" ) ) else: self._validate_items_list(items, f"{section_name}.items_to_include") elif isinstance(items, dict): # Validate environment mapping if not self._validate_environment_mapping(items, f"{section_name}.items_to_include", list): return # Validate each environment's items list for env, items_list in items.items(): self._validate_items_list(items_list, f"{section_name}.items_to_include.{env}") else: self.errors.append( constants.CONFIG_VALIDATION_MSGS["field"]["list_or_dict"].format( f"{section_name}.items_to_include", type(items).__name__ ) ) # Validate folder_exclude_regex if present (publish only) if "folder_exclude_regex" in section: if section_name != "publish": self.errors.append( constants.CONFIG_VALIDATION_MSGS["operation"]["unsupported_field"].format( "folder_exclude_regex", section_name ) ) folder_exclude_regex = section["folder_exclude_regex"] if isinstance(folder_exclude_regex, str): if not folder_exclude_regex.strip(): self.errors.append( constants.CONFIG_VALIDATION_MSGS["operation"]["empty_string"].format( f"{section_name}.folder_exclude_regex" ) ) else: self._validate_regex(folder_exclude_regex, f"{section_name}.folder_exclude_regex") elif isinstance(folder_exclude_regex, dict): # Validate environment mapping if not self._validate_environment_mapping( folder_exclude_regex, f"{section_name}.folder_exclude_regex", str ): return # Validate each environment's regex pattern for env, regex_pattern in folder_exclude_regex.items(): self._validate_regex(regex_pattern, f"{section_name}.folder_exclude_regex.{env}") else: self.errors.append( constants.CONFIG_VALIDATION_MSGS["field"]["string_or_dict"].format( f"{section_name}.folder_exclude_regex", type(folder_exclude_regex).__name__ ) ) # Validate folder_path_to_include if present (publish only) if "folder_path_to_include" in section: if section_name != "publish": self.errors.append( constants.CONFIG_VALIDATION_MSGS["operation"]["unsupported_field"].format( "folder_path_to_include", section_name ) ) folders = section["folder_path_to_include"] if isinstance(folders, list): if not folders: self.errors.append( constants.CONFIG_VALIDATION_MSGS["operation"]["empty_list"].format( f"{section_name}.folder_path_to_include" ) ) else: self._validate_folders_list(folders, f"{section_name}.folder_path_to_include") elif isinstance(folders, dict): # Validate environment mapping if not self._validate_environment_mapping(folders, f"{section_name}.folder_path_to_include", list): return # Validate each environment's folders list for env, folders_list in folders.items(): self._validate_folders_list(folders_list, f"{section_name}.folder_path_to_include.{env}") else: self.errors.append( constants.CONFIG_VALIDATION_MSGS["field"]["list_or_dict"].format( f"{section_name}.folder_path_to_include", type(folders).__name__ ) ) # Validate shortcut_exclude_regex if present (publish only) if "shortcut_exclude_regex" in section: if section_name != "publish": self.errors.append( f"'{section_name}.shortcut_exclude_regex' is not supported - shortcut exclusion is only available in the 'publish' section" ) shortcut_exclude_regex = section["shortcut_exclude_regex"] if isinstance(shortcut_exclude_regex, str): if not shortcut_exclude_regex.strip(): self.errors.append( constants.CONFIG_VALIDATION_MSGS["operation"]["empty_string"].format( f"{section_name}.shortcut_exclude_regex" ) ) else: self._validate_regex(shortcut_exclude_regex, f"{section_name}.shortcut_exclude_regex") elif isinstance(shortcut_exclude_regex, dict): # Validate environment mapping if not self._validate_environment_mapping( shortcut_exclude_regex, f"{section_name}.shortcut_exclude_regex", str ): return # Validate each environment's regex pattern for env, regex_pattern in shortcut_exclude_regex.items(): self._validate_regex(regex_pattern, f"{section_name}.shortcut_exclude_regex.{env}") else: self.errors.append( constants.CONFIG_VALIDATION_MSGS["field"]["string_or_dict"].format( f"{section_name}.shortcut_exclude_regex", type(shortcut_exclude_regex).__name__ ) ) # Validate skip if present if "skip" in section: skip_value = section["skip"] if isinstance(skip_value, bool): # Single boolean value return if isinstance(skip_value, dict): # Use the reusable environment mapping validation if not self._validate_environment_mapping(skip_value, f"{section_name}.skip", bool): return else: self.errors.append( constants.CONFIG_VALIDATION_MSGS["field"]["string_or_dict"] .format(f"{section_name}.skip", type(skip_value).__name__) .replace("a string", "a boolean") ) # Validate mutual exclusivity of folder filtering options self._validate_mutually_exclusive_fields( section, "folder_exclude_regex", "folder_path_to_include", section_name ) def _validate_regex(self, regex: str, section_name: str) -> None: """Validate regex value.""" try: re.compile(regex) except re.error as e: self.errors.append( constants.CONFIG_VALIDATION_MSGS["operation"]["invalid_regex"].format(regex, section_name, e) ) def _validate_items_list(self, items_list: list, context: str) -> None: """Validate a list of items with proper context for error messages.""" for i, item in enumerate(items_list): if not isinstance(item, str): self.errors.append( constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_type"].format( context, i, type(item).__name__ ) ) elif not item.strip(): self.errors.append(constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_empty"].format(context, i)) def _validate_folders_list(self, folders_list: list, context: str) -> None: """Validate a list of folder paths with proper context for error messages.""" for i, folder in enumerate(folders_list): if not isinstance(folder, str): self.errors.append( constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_type"].format( context, i, type(folder).__name__ ) ) elif not folder.strip(): self.errors.append(constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_empty"].format(context, i)) elif not folder.startswith("/"): self.errors.append( constants.CONFIG_VALIDATION_MSGS["operation"]["folders_list_prefix"].format(context, i, folder) ) def _validate_mutually_exclusive_fields(self, section: dict, field1: str, field2: str, section_name: str) -> None: """Validate that two fields are not both specified for the same environment.""" if field1 not in section or field2 not in section: return value1 = section[field1] value2 = section[field2] # Both are direct values (not environment-specific), throw error if not isinstance(value1, dict) and not isinstance(value2, dict): self.errors.append( constants.CONFIG_VALIDATION_MSGS["operation"]["mutually_exclusive"].format( f"{section_name}.{field1}", f"{section_name}.{field2}" ) ) return # Determine which environments each field contains (if they are environment mappings) value1_envs = set(value1.keys()) if isinstance(value1, dict) else set() value2_envs = set(value2.keys()) if isinstance(value2, dict) else set() # Determine if it is a direct value value1_is_direct = not isinstance(value1, dict) value2_is_direct = not isinstance(value2, dict) # Check if both fields would resolve for the target environment value1_applies = value1_is_direct or self.environment in value1_envs value2_applies = value2_is_direct or self.environment in value2_envs if value1_applies and value2_applies: self.errors.append( constants.CONFIG_VALIDATION_MSGS["operation"]["mutually_exclusive_env"].format( f"{section_name}.{field1}", f"{section_name}.{field2}", [self.environment], ) ) def _validate_features_section(self, features: any) -> None: """Validate features section.""" if isinstance(features, list): if not features: self.errors.append(constants.CONFIG_VALIDATION_MSGS["operation"]["empty_section"].format("features")) return self._validate_features_list(features, "features") return if isinstance(features, dict): # Validate environment mapping if not self._validate_environment_mapping(features, "features", list): return # Validate each environment's features list for env, features_list in features.items(): self._validate_features_list(features_list, f"features.{env}") return self.errors.append( constants.CONFIG_VALIDATION_MSGS["operation"]["features_type"].format(type(features).__name__) ) def _validate_features_list(self, features_list: list, context: str) -> None: """Validate a list of features with proper context for error messages.""" for i, feature in enumerate(features_list): if not isinstance(feature, str): self.errors.append( constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_type"].format( context, i, type(feature).__name__ ) ) elif not feature.strip(): self.errors.append(constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_empty"].format(context, i)) def _validate_constants_section(self, constants_section: any) -> None: """Validate constants section.""" if not isinstance(constants_section, dict): self.errors.append( constants.CONFIG_VALIDATION_MSGS["operation"]["not_dict"].format( "constants", type(constants_section).__name__ ) ) return # Validate each constant key individually — values can be flat or per-key env mappings for key, value in constants_section.items(): if not isinstance(key, str) or not key.strip(): self.errors.append( constants.CONFIG_VALIDATION_MSGS["operation"]["invalid_constant_key"].format("constants", key) ) continue # Validate that the constant exists in the constants module if not hasattr(constants, key): self.errors.append( constants.CONFIG_VALIDATION_MSGS["operation"]["unknown_constant"].format(key, "constants") ) if isinstance(value, dict): # Per-key environment mapping: { KEY: { dev: val, prod: val } } for env, env_value in value.items(): if not isinstance(env, str) or not env.strip(): self.errors.append( constants.CONFIG_VALIDATION_MSGS["environment"]["invalid_env_key"].format( f"constants.{key}", type(env).__name__ ) ) continue self._validate_single_constant(key, env_value, f"constants.{key}.{env}") else: # Flat value: { KEY: val } self._validate_single_constant(key, value, f"constants.{key}") def _validate_single_constant(self, key: str, value: any, context: str) -> None: """Validate a single constant value.""" if key in _URL_CONSTANTS: if not isinstance(value, str): self.errors.append(f"'{context}' must be a string URL, got {type(value).__name__}") return try: validate_api_url(value, context) except InputError as e: self.errors.append(str(e)) def _get_config_fields(config: dict) -> list[tuple[dict, str, str, bool, bool]]: """Get list of all fields that support environment mappings. Returns: List of tuples: (section_dict, field_name, display_name, is_required, log_warning) - is_required: If True, missing environment causes error. - log_warning: logging type (e.g., warning (True), debug (False)). """ return [ # Core section fields - required (config.get("core", {}), "workspace_id", "core.workspace_id", True, False), (config.get("core", {}), "workspace", "core.workspace", True, False), (config.get("core", {}), "repository_directory", "core.repository_directory", True, False), # Core section fields - optional but important (warn if missing) (config.get("core", {}), "item_types_in_scope", "core.item_types_in_scope", False, True), (config.get("core", {}), "parameter", "core.parameter", False, True), # Publish section fields - optional (debug if missing) (config.get("publish", {}), "exclude_regex", "publish.exclude_regex", False, False), (config.get("publish", {}), "folder_exclude_regex", "publish.folder_exclude_regex", False, False), (config.get("publish", {}), "folder_path_to_include", "publish.folder_path_to_include", False, False), (config.get("publish", {}), "shortcut_exclude_regex", "publish.shortcut_exclude_regex", False, False), (config.get("publish", {}), "items_to_include", "publish.items_to_include", False, False), (config.get("publish", {}), "skip", "publish.skip", False, False), # Unpublish section fields - optional (debug if missing) (config.get("unpublish", {}), "exclude_regex", "unpublish.exclude_regex", False, False), (config.get("unpublish", {}), "items_to_include", "unpublish.items_to_include", False, False), (config.get("unpublish", {}), "skip", "unpublish.skip", False, False), # Top-level sections - optional (warn if missing) (config, "features", "features", False, True), (config, "constants", "constants", False, True), ] def _find_git_root(path: Path) -> Optional[Path]: """Find the git repository root for a given path.""" current = path.resolve() while current != current.parent: if (current / ".git").exists(): return current current = current.parent return None def _validate_guid_format(guid: str) -> bool: """Validate GUID format using the pattern from constants.""" return bool(re.match(constants.VALID_GUID_REGEX, guid)) ================================================ FILE: src/fabric_cicd/_common/_deployment_result.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Deployment result types for config-based deployment operations.""" from dataclasses import dataclass, field from enum import Enum from typing import Optional class DeploymentStatus(str, Enum): """Enumeration of deployment status values for deploy_with_config results.""" COMPLETED = "completed" """Deployment completed successfully without any errors.""" FAILED = "failed" """Deployment failed due to one or more errors.""" @dataclass class DeploymentResult: """Structured result of a config-based deployment operation. Returned by ``deploy_with_config`` on success. On failure, an instance is attached to the raised exception as ``e.deployment_result`` with ``status`` set to ``DeploymentStatus.FAILED``. Attributes: status: The deployment status. message: A human-readable message describing the result. responses: Optional dictionary of API response data from the deployment. """ status: DeploymentStatus message: str responses: Optional[dict] = field(default=None) ================================================ FILE: src/fabric_cicd/_common/_exceptions.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Custom exceptions for the fabric-cicd package.""" from logging import Logger from typing import Optional class BaseCustomError(Exception): def __init__(self, message: str, logger: Logger, additional_info: Optional[str] = None) -> None: """ Initialize the BaseCustomError. Args: message: The error message. logger: The logger instance. additional_info: Additional information about the error. """ super().__init__(message) self.logger = logger self.additional_info = additional_info class ParsingError(BaseCustomError): pass class InputError(BaseCustomError): pass class TokenError(BaseCustomError): pass class InvokeError(BaseCustomError): pass class ItemDependencyError(BaseCustomError): pass class FileTypeError(BaseCustomError): pass class ParameterFileError(BaseCustomError): pass class FailedPublishedItemStatusError(BaseCustomError): pass class PublishError(BaseCustomError): """Exception raised when one or more publish operations fail. Attributes: errors: List of (item_name, exception) tuples for all failed items. """ def __init__(self, errors: list[tuple[str, Exception]], logger: Logger) -> None: """Initialize with a list of (item_name, exception) tuples.""" self.errors = errors failed_names = [name for name, _ in errors] message = f"Failed to publish {len(errors)} item(s): {failed_names}" additional_info_parts = [] for item_name, exc in errors: additional_info_parts.append(f"\n--- {item_name} ---\n{exc!s}") additional_info = "\n".join(additional_info_parts) if additional_info_parts else None super().__init__(message, logger, additional_info) ================================================ FILE: src/fabric_cicd/_common/_fabric_endpoint.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Handles interactions with the Fabric API, including authentication and request management.""" import datetime import json import logging import os import time from typing import Optional import requests from azure.core.credentials import TokenCredential from azure.core.exceptions import ( ClientAuthenticationError, ) import fabric_cicd.constants as constants from fabric_cicd._common._exceptions import InvokeError, TokenError from fabric_cicd._common._http_tracer import HTTPTracer, HTTPTracerFactory logger = logging.getLogger(__name__) class FabricEndpoint: """Handles interactions with the Fabric API, including authentication and request management.""" def __init__( self, token_credential: TokenCredential, requests_module: requests = requests, http_tracer: Optional[HTTPTracer] = None, ) -> None: """ Initializes the FabricEndpoint instance, sets up the authentication token. Args: token_credential: The token credential. requests_module: The requests module. http_tracer: Optional HTTP tracer for debugging. If None, create using factory. """ self.aad_token = None self.aad_token_expiration = None self.token_credential = token_credential self.requests = requests_module self.http_tracer = http_tracer if http_tracer is not None else HTTPTracerFactory.create() self._refresh_token() def invoke( self, method: str, url: str, body: str = "{}", files: Optional[dict] = None, poll_long_running: bool = True, max_duration: int = 300, **kwargs, ) -> dict: """ Sends an HTTP request to the specified URL with the given method and body. Args: method: HTTP method to use for the request (e.g., 'GET', 'POST', 'PATCH', 'DELETE'). url: URL to send the request to. body: The JSON body to include in the request. Defaults to an empty JSON object. files: The file path to be included in the request. Defaults to None. poll_long_running: A flag to poll for long-running operations. Defaults to True. max_duration: Maximum execution duration in seconds. Defaults to 300 (5 minutes). **kwargs: Additional keyword arguments to pass to the method. """ exit_loop = False iteration_count = 0 long_running = False start_time = time.time() invoke_log_message = "" while not exit_loop: try: headers = { "Authorization": f"Bearer {self.aad_token}", "User-Agent": f"{constants.USER_AGENT}", } if files is None: headers["Content-Type"] = "application/json; charset=utf-8" self.http_tracer.capture_request(method, url, headers, body, files) response = self.requests.request(method=method, url=url, headers=headers, json=body, files=files) self.http_tracer.capture_response(response) iteration_count += 1 invoke_log_message = _format_invoke_log(response, method, url, body) # Handle expired authentication token if response.status_code == 401 and response.headers.get("x-ms-public-api-error-code") == "TokenExpired": logger.info(f"{constants.INDENT}AAD token expired. Refreshing token.") self._refresh_token() # Handle long-running operations without polling (e.g., for environment item publish) elif response.status_code == 202 and not poll_long_running: # Accept 202, do not poll exit_loop = True else: exit_loop, method, url, body, long_running = _handle_response( response, method, url, body, long_running, iteration_count, max_duration, start_time, **kwargs, ) # Log if reached to end of loop iteration if logger.isEnabledFor(logging.DEBUG): logger.debug(invoke_log_message) except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: iteration_count += 1 if max_duration is not None and time.time() - start_time >= max_duration: logger.debug(invoke_log_message) raise InvokeError(e, logger, invoke_log_message) from e handle_retry( attempt=iteration_count, base_delay=10, max_duration=max_duration, start_time=start_time, prepend_message="Connection error encountered.", ) except Exception as e: logger.debug(invoke_log_message) raise InvokeError(e, logger, invoke_log_message) from e end_time = time.time() logger.debug(f"Request completed in {end_time - start_time} seconds") if exit_loop: self.http_tracer.save() return { "header": dict(response.headers), "body": (response.json() if "application/json" in response.headers.get("Content-Type") else {}), "status_code": response.status_code, } def _refresh_token(self) -> None: """Refreshes the AAD token if empty or expiration has passed.""" if ( self.aad_token is None or self.aad_token_expiration is None or self.aad_token_expiration < datetime.datetime.now(datetime.timezone.utc) ): resource_url = "https://api.fabric.microsoft.com/.default" try: access_token = self.token_credential.get_token(resource_url) self.aad_token = access_token.token self.aad_token_expiration = datetime.datetime.fromtimestamp( access_token.expires_on, tz=datetime.timezone.utc ) except ClientAuthenticationError as e: msg = f"Failed to acquire AAD token. {e}" raise TokenError(msg, logger) from e except Exception as e: msg = f"An unexpected error occurred when generating the AAD token. {e}" raise TokenError(msg, logger) from e def _handle_response( response: requests.Response, method: str, url: str, body: str, long_running: bool, iteration_count: int, max_duration: Optional[int] = None, start_time: Optional[float] = None, ) -> tuple: """ Handles the response from an HTTP request, including retries, throttling, and token expiration. Technical debt: this method needs to be refactored to be more testable and requires less parameters. Initial approach is only temporary to support testing, but only temporary. Args: response: The response object from the HTTP request. method: The HTTP method used in the request. url: The URL used in the request. body: The JSON body used in the request. long_running: A boolean indicating if the operation is long-running. iteration_count: The current iteration count of the loop. max_duration: Maximum execution duration in seconds. Defaults to None. start_time: The start time of the request in seconds since epoch. Defaults to None. """ exit_loop = False retry_after = response.headers.get("Retry-After", 60) # Handle long-running operations # https://learn.microsoft.com/en-us/rest/api/fabric/core/long-running-operations/get-operation-result if (response.status_code == 200 and long_running) or response.status_code == 202: url = response.headers.get("Location") method = "GET" body = "{}" response_json = response.json() if response.text else {} if long_running: status = response_json.get("status") if status == "Succeeded": long_running = False # If location not included in operation success call, no body is expected to be returned exit_loop = url is None elif status == "Failed": response_error = response_json["error"] msg = ( f"Operation failed. Error Code: {response_error['errorCode']}. " f"Error Message: {response_error['message']}" ) raise Exception(msg) elif status == "Undefined": msg = f"Operation is in an undefined state. Full Body: {response_json}" raise Exception(msg) else: handle_retry( attempt=iteration_count - 1, base_delay=0.5, response_retry_after=retry_after, max_duration=max_duration, start_time=start_time, prepend_message=f"{constants.INDENT}Operation in progress.", ) else: if url is None: # No Location header means operation completed immediately exit_loop = True else: # We check for system level config for retry delay override, e.g. in unit # tests where we want to rip through thousands of API calls quickly. If not # set, e.g. at runtime, we use normal polling delay of default 1 second. time.sleep(float(os.environ.get(constants.EnvVar.RETRY_DELAY_OVERRIDE_SECONDS.value, 1))) long_running = True # Handle successful responses elif response.status_code in {200, 201} or ( # Valid response for environmentlibrariesnotfound response.status_code == 404 and response.headers.get("x-ms-public-api-error-code") == "EnvironmentLibrariesNotFound" ): exit_loop = True # Handle API throttling elif response.status_code == 429: handle_retry( attempt=iteration_count, base_delay=10, max_duration=max_duration, start_time=start_time, response_retry_after=retry_after, prepend_message="API is throttled.", ) # Handle internal server errors via retry, # rather than failing the deployment run elif response.status_code == 500: handle_retry( attempt=iteration_count, base_delay=10, max_duration=max_duration, start_time=start_time, response_retry_after=retry_after, prepend_message="Server error encountered.", ) # Handle unauthorized access elif response.status_code == 401 and response.headers.get("x-ms-public-api-error-code") == "Unauthorized": msg = f"The executing identity is not authorized to call {method} on '{url}'." raise Exception(msg) # Handle item name conflicts elif ( response.status_code == 400 and response.headers.get("x-ms-public-api-error-code") == "ItemDisplayNameNotAvailableYet" ): handle_retry( attempt=iteration_count, base_delay=constants.RETRY_BASE_DELAY_SECONDS, max_duration=constants.RETRY_MAX_DURATION_SECONDS, start_time=start_time, response_retry_after=constants.RETRY_AFTER_SECONDS, prepend_message="Item name is reserved.", ) # Handle scenario where library removed from environment before being removed from repo elif response.status_code == 400 and "is not present in the environment." in response.json().get( "message", "No message provided" ): msg = ( f"Deployment attempted to remove a library that is not present in the environment. " f"Description: {response.json().get('message')}" ) raise Exception(msg) # Handle unsupported principal type elif ( response.status_code == 400 and response.headers.get("x-ms-public-api-error-code") == "PrincipalTypeNotSupported" ): msg = f"The executing principal type is not supported to call {method} on '{url}'." raise Exception(msg) # Handle unsupported item types elif response.status_code == 403 and response.reason == "FeatureNotAvailable": msg = f"Item type not supported. Description: {response.reason}" raise Exception(msg) # Handle unexpected errors else: err_msg = ( f" Message: {response.json()['message']}. {response.json().get('moreDetails', '')}" if "application/json" in (response.headers.get("Content-Type") or "") else "" ) msg = f"Unhandled error occurred calling {method} on '{url}'.{err_msg}" raise Exception(msg) return exit_loop, method, url, body, long_running def handle_retry( attempt: int, base_delay: float, response_retry_after: float = 60, prepend_message: str = "", max_duration: Optional[int] = None, start_time: Optional[float] = None, ) -> None: """ Handles retry logic with exponential backoff based on the response, retrying until the maximum duration is reached. Args: attempt: The current attempt number. base_delay: Base delay in seconds for backoff. response_retry_after: The value of the Retry-After header from the response. prepend_message: Message to prepend to the retry log. max_duration: Maximum execution duration in seconds. If None, retries indefinitely. start_time: The start time of the request in seconds since epoch. Required if max_duration is set. """ if max_duration is None or (start_time is not None and time.time() - start_time < max_duration): retry_delay_override = os.environ.get(constants.EnvVar.RETRY_DELAY_OVERRIDE_SECONDS.value) if retry_delay_override is not None: delay = float(retry_delay_override) else: retry_after = float(response_retry_after) base_delay = float(base_delay) delay = min(retry_after, base_delay * (2**attempt)) # modify output for proper plurality and formatting delay_str = f"{delay:.0f}" if delay.is_integer() else f"{delay:.2f}" second_str = "second" if delay == 1 else "seconds" prepend_message += " " if prepend_message else "" logger.info( f"{constants.INDENT}{prepend_message}Checking again in {delay_str} {second_str} (Attempt {attempt})..." ) time.sleep(delay) else: elapsed = time.time() - start_time if start_time is not None else 0 msg = f"Maximum execution duration ({max_duration} seconds) exceeded after {elapsed:.1f} seconds." raise Exception(msg) def _format_invoke_log(response: requests.Response, method: str, url: str, body: str) -> str: """ Format the log message for the invoke method. Args: response: The response object from the HTTP request. method: The HTTP method used in the request. url: The URL used in the request. body: The JSON body used in the request. """ message = [ f"\nURL: {url}", f"Method: {method}", (f"Request Body:\n{json.dumps(body, indent=4)}" if body else "Request Body: None"), ] if response is not None: message.extend([ f"Response Status: {response.status_code}", "Response Headers:", json.dumps(dict(response.headers), indent=4), "Response Body:", ( json.dumps(response.json(), indent=4) if response.headers.get("Content-Type") == "application/json" else response.text ), "", ]) return "\n".join(message) ================================================ FILE: src/fabric_cicd/_common/_file.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions and classes to manage file operations.""" import base64 import logging from dataclasses import dataclass, field from pathlib import Path from typing import ClassVar from fabric_cicd._common._check_utils import check_file_type from fabric_cicd._common._exceptions import FileTypeError logger = logging.getLogger(__name__) @dataclass() class File: """A class to represent a single file in an item object.""" item_path: Path file_path: Path type: str = field(default="text", init=False) contents: str = field(default="", init=False) IMMUTABLE_FIELDS: ClassVar[set] = {"item_path", "file_path"} def __setattr__(self, key: str, value: any) -> None: """ Override setattr for 'immutable' fields. Args: key: The attribute name. value: The attribute value. """ if key in self.IMMUTABLE_FIELDS and hasattr(self, key): msg = f"item {key} is immutable" raise AttributeError(msg) # Image file contents cannot be set if key == "contents" and self.type != "text": msg = f"item {key} is immutable for non text files" raise AttributeError(msg) super().__setattr__(key, value) def __post_init__(self) -> None: """After initializing the object, read the file contents and set the type.""" file_type = check_file_type(self.file_path) if file_type != "text": try: self.contents = self.file_path.read_bytes() except Exception as e: msg = ( f"Error reading file {self.file_path} as binary. " f"Please submit this as a bug https://github.com/microsoft/fabric-cicd/issues/new?template=1-bug.yml.md. Exception: {e}" ) FileTypeError(msg, logger) else: try: self.contents = self.file_path.read_text(encoding="utf-8") except Exception as e: msg = ( f"Error reading file {self.file_path} as text. " f"Please submit this as a bug https://github.com/microsoft/fabric-cicd/issues/new?template=1-bug.yml.md. Exception: {e}" ) FileTypeError(msg, logger) # set after as image contents are now immutable self.type = file_type @property def name(self) -> str: """Return the file name.""" return self.file_path.name @property def relative_path(self) -> str: """Return the relative path of the file.""" return str(self.file_path.relative_to(self.item_path).as_posix()) @property def base64_payload(self) -> dict: """Return the file contents as a base64 encoded payload.""" byte_file = self.contents.encode("utf-8") if self.type == "text" else self.contents return { "path": self.relative_path, "payload": base64.b64encode(byte_file).decode("utf-8"), "payloadType": "InlineBase64", } ================================================ FILE: src/fabric_cicd/_common/_file_lock.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Cross-platform file locking.""" import sys from pathlib import Path from types import TracebackType from typing import Callable, Optional, TypeVar T = TypeVar("T") class FileLock: """File lock context manager.""" def __init__(self, lock_file: str) -> None: self.lock_path = Path(f"{lock_file}.lock") self._lock_file: Optional[object] = None def __enter__(self) -> "FileLock": self._lock_file = self.lock_path.open("w") if sys.platform == "win32": import msvcrt msvcrt.locking(self._lock_file.fileno(), msvcrt.LK_LOCK, 1) else: import fcntl fcntl.flock(self._lock_file.fileno(), fcntl.LOCK_EX) return self def __exit__( self, exc_type: Optional[type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType], ) -> bool: if self._lock_file: if sys.platform == "win32": import msvcrt msvcrt.locking(self._lock_file.fileno(), msvcrt.LK_UNLCK, 1) else: import fcntl fcntl.flock(self._lock_file.fileno(), fcntl.LOCK_UN) self._lock_file.close() return False @staticmethod def run_with_lock(lock_file: str, func: Callable[[], T]) -> T: """ Execute a function while holding an exclusive file lock. Args: lock_file: Path to the file to lock (a .lock suffix will be added) func: The function to execute while holding the lock Returns: The return value of the function """ with FileLock(lock_file): return func() ================================================ FILE: src/fabric_cicd/_common/_git_diff_utils.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Utility functions for detecting Fabric items changed via git diff.""" import json import logging import subprocess from pathlib import Path from typing import Optional logger = logging.getLogger(__name__) def _find_platform_item(file_path: Path, repo_root: Path) -> Optional[tuple[str, str]]: """ Walk up from file_path towards repo_root looking for a .platform file. The .platform file marks the boundary of a Fabric item directory. Its JSON content contains ``metadata.type`` (item type) and ``metadata.displayName`` (item name). Returns: A ``(item_name, item_type)`` tuple, or ``None`` if not found. """ current = file_path.parent while True: platform_file = current / ".platform" if platform_file.exists(): try: data = json.loads(platform_file.read_text(encoding="utf-8")) metadata = data.get("metadata", {}) item_type = metadata.get("type") item_name = metadata.get("displayName") or current.name if item_type: return item_name, item_type except Exception as exc: logger.debug(f"Could not parse .platform file at '{platform_file}': {exc}") # Stop if we have reached the repository root or the filesystem root if current == repo_root or current == current.parent: break current = current.parent return None def _resolve_git_diff_path( file_path_str: str, git_root: Path, repository_directory: Path, ) -> Optional[Path]: """ Resolve and validate a file path from git diff output. Follows the same resolve → boundary-check → reject contract as ``_resolve_file_path`` in ``_parameter/_utils.py``, adapted for paths that are relative to a git root with containment checked against a (potentially different) repository subdirectory. Args: file_path_str: Relative path string from git diff output. git_root: Resolved absolute path of the git repository root. repository_directory: Resolved absolute path of the configured repository directory (may be a subdirectory of git_root). Returns: Resolved absolute Path if valid and within boundary, None otherwise. """ raw_path = Path(file_path_str) # Reject absolute paths — git diff should only produce relative paths if raw_path.is_absolute(): logger.debug(f"get_changed_items: skipping absolute path '{file_path_str}'") return None # Reject traversal sequences before resolution (mirrors _validate_wildcard_syntax) if ".." in raw_path.parts: logger.debug(f"get_changed_items: skipping path with traversal '{file_path_str}'") return None # Reject null bytes if "\x00" in file_path_str: logger.debug("get_changed_items: skipping path with null bytes") return None # Step 1: Resolve relative to git root (analogous to _resolve_file_path Step 1) resolved_path = (git_root / file_path_str).resolve() # Step 2: Boundary check against repository_directory (analogous to _resolve_file_path Step 2) try: resolved_path.relative_to(repository_directory) except ValueError: return None # Note: No Step 3 (existence check) — deleted files won't exist on disk return resolved_path def get_changed_items( repository_directory: Path, git_compare_ref: str = "HEAD~1", ) -> list[str]: """ Return the list of Fabric items that were added, modified, or renamed relative to ``git_compare_ref``. The returned list is in ``"item_name.item_type"`` format and can be passed directly to the ``items_to_include`` parameter of :func:`publish_all_items` to deploy only what has changed since the last commit. Args: repository_directory: Path to the local git repository directory (e.g. ``FabricWorkspace.repository_directory``). git_compare_ref: Git ref to compare against. Defaults to ``"HEAD~1"``. Returns: List of strings in ``"item_name.item_type"`` format. Returns an empty list when no changes are detected, the git root cannot be found, or git is unavailable. Examples: Deploy only changed items >>> from azure.identity import AzureCliCredential >>> from fabric_cicd import FabricWorkspace, publish_all_items, get_changed_items >>> workspace = FabricWorkspace( ... workspace_id="your-workspace-id", ... repository_directory="/path/to/repo", ... item_type_in_scope=["Notebook", "DataPipeline"], ... token_credential=AzureCliCredential() ... ) >>> changed = get_changed_items(workspace.repository_directory) >>> if changed: ... publish_all_items(workspace, items_to_include=changed) With a custom git ref >>> changed = get_changed_items(workspace.repository_directory, git_compare_ref="main") >>> if changed: ... publish_all_items(workspace, items_to_include=changed) """ changed, _ = _resolve_changed_items(Path(repository_directory), git_compare_ref) return changed def _resolve_changed_items( repository_directory: Path, git_compare_ref: str, ) -> tuple[list[str], list[str]]: """ Use ``git diff --name-status`` to detect Fabric items that changed or were deleted relative to *git_compare_ref*. Args: repository_directory: Absolute path to the local repository directory (as stored on ``FabricWorkspace.repository_directory``). git_compare_ref: Git ref to diff against (e.g. ``"HEAD~1"``). Returns: A two-element tuple ``(changed_items, deleted_items)`` where each element is a list of strings in ``"item_name.item_type"`` format. Both lists are empty when the git root cannot be found or git fails. """ from fabric_cicd._common._config_validator import _find_git_root from fabric_cicd._common._validate_input import validate_git_compare_ref validate_git_compare_ref(git_compare_ref) git_root = _find_git_root(repository_directory) if git_root is None: logger.warning("get_changed_items: could not locate a git repository root — returning empty list.") return [], [] try: result = subprocess.run( ["git", "diff", "--name-status", git_compare_ref], cwd=str(git_root), capture_output=True, text=True, check=True, timeout=30, ) except subprocess.CalledProcessError as exc: logger.warning(f"get_changed_items: 'git diff' failed ({exc.stderr.strip()}) — returning empty list.") return [], [] except subprocess.TimeoutExpired: logger.warning("get_changed_items: 'git diff' timed out — returning empty list.") return [], [] changed_items: set[str] = set() deleted_items: set[str] = set() git_root_resolved = git_root.resolve() repo_dir_resolved = repository_directory.resolve() for line in result.stdout.splitlines(): line = line.strip() if not line: continue parts = line.split("\t") status = parts[0].strip() # Renames produce three tab-separated fields: R\told\tnew if status.startswith("R") and len(parts) >= 3: file_path_str = parts[2] elif len(parts) >= 2: file_path_str = parts[1] else: continue abs_path = _resolve_git_diff_path(file_path_str, git_root_resolved, repo_dir_resolved) if abs_path is None: continue if status == "D": if abs_path.name == ".platform": try: show_result = subprocess.run( ["git", "show", f"{git_compare_ref}:{file_path_str}"], cwd=str(git_root_resolved), capture_output=True, text=True, check=True, timeout=30, ) data = json.loads(show_result.stdout) metadata = data.get("metadata", {}) item_type = metadata.get("type") item_name = metadata.get("displayName") or abs_path.parent.name if item_type and item_name: deleted_items.add(f"{item_name}.{item_type}") except Exception as exc: logger.debug(f"get_changed_items: could not read deleted .platform '{file_path_str}': {exc}") else: item_info = _find_platform_item(abs_path, repo_dir_resolved) if item_info: changed_items.add(f"{item_info[0]}.{item_info[1]}") return list(changed_items), list(deleted_items) ================================================ FILE: src/fabric_cicd/_common/_http_tracer.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """HTTP request/response tracer for debugging and mock server generation.""" import base64 import hashlib import json import logging import os import uuid from dataclasses import asdict, dataclass from datetime import date, datetime, timezone from pathlib import Path from typing import Any, Optional, Protocol from urllib.parse import urlparse import requests from fabric_cicd._common._file_lock import FileLock from fabric_cicd.constants import AUTHORIZATION_HEADER, EnvVar logger = logging.getLogger(__name__) def _trace_default(obj: object) -> str: """Deterministic JSON fallback for non-native objects in HTTP traces.""" if isinstance(obj, (datetime, date)): return obj.isoformat() if isinstance(obj, uuid.UUID): return str(obj) if isinstance(obj, (bytes, bytearray)): try: return bytes(obj).decode("utf-8") except UnicodeDecodeError: return base64.b64encode(bytes(obj)).decode("ascii") return f"" @dataclass class HTTPRequest: """Represents an HTTP request with metadata.""" method: str url: str headers: dict[str, str] body: Any timestamp: str def to_b64(self) -> str: """Serialize to base64-encoded JSON.""" request_json = json.dumps(asdict(self), separators=(",", ":"), default=_trace_default) return base64.b64encode(request_json.encode()).decode() @classmethod def from_b64(cls, b64_str: str) -> "HTTPRequest": """Deserialize from base64-encoded JSON.""" json_str = base64.b64decode(b64_str).decode() data = json.loads(json_str) return cls(**data) def get_unique_signature(self) -> str: """Generate unique signature from URL, method, and body using SHA256.""" body_str = json.dumps(self.body, sort_keys=True) if isinstance(self.body, dict) else str(self.body or "") return hashlib.sha256(f"{self.url}{self.method}{body_str}".encode()).hexdigest() def get_route_key(self) -> str: """Extract route key (method + path + query) from the request.""" try: parsed_url = urlparse(self.url) route = parsed_url.path if parsed_url.query: route += f"?{parsed_url.query}" return f"{self.method} {route}" except Exception: return "" @dataclass class HTTPResponse: """Represents an HTTP response with metadata.""" status_code: int headers: dict[str, str] body: Any timestamp: str def to_b64(self) -> str: """Serialize to base64-encoded JSON.""" response_json = json.dumps(asdict(self), separators=(",", ":"), default=_trace_default) return base64.b64encode(response_json.encode()).decode() @classmethod def from_b64(cls, b64_str: str) -> "HTTPResponse": """Deserialize from base64-encoded JSON.""" json_str = base64.b64decode(b64_str).decode() data = json.loads(json_str) return cls(**data) def get_unique_signature(self) -> str: """Generate unique signature from status code and body using SHA256.""" body_str = json.dumps(self.body, sort_keys=True) if isinstance(self.body, dict) else str(self.body or "") return hashlib.sha256(f"{self.status_code}{body_str}".encode()).hexdigest() class HTTPTracer(Protocol): """Protocol for HTTP request/response tracers.""" def capture_request(self, method: str, url: str, headers: dict, body: str, files: Optional[dict]) -> None: """Capture HTTP request details.""" ... def capture_response(self, response: requests.Response) -> None: """Capture HTTP response details.""" ... def save(self) -> None: """Save captured data.""" ... class NoOpTracer: """No-op tracer that does nothing.""" def capture_request(self, method: str, url: str, headers: dict, body: str, files: Optional[dict]) -> None: """No-op capture request.""" pass def capture_response(self, response: requests.Response) -> None: """No-op capture response.""" pass def save(self) -> None: """No-op save.""" pass class FileTracer: """Captures HTTP requests and responses to a JSON file.""" def __init__(self, output_file: Optional[str] = None) -> None: """ Initialize the file tracer. Args: output_file: Path to save the trace file. If None, checks EnvVar.HTTP_TRACE_FILE. """ trace_file_from_env = os.environ.get(EnvVar.HTTP_TRACE_FILE.value) if output_file is None: self.output_file = trace_file_from_env if trace_file_from_env else "http_trace.json" else: self.output_file = output_file self.captures: list[dict] = [] def capture_request(self, method: str, url: str, headers: dict, body: str, files: Optional[dict]) -> None: # noqa: ARG002 """ Capture HTTP request details with base64 encoding, if enabled. Args: method: HTTP method (GET, POST, etc.) url: Request URL headers: Request headers. Note: Authorization headers are excluded. body: Request body files: Files being uploaded """ request = HTTPRequest( method=method, url=url, headers={k: v for k, v in headers.items() if k.lower() not in [AUTHORIZATION_HEADER]}, body=body, timestamp=datetime.now(timezone.utc).isoformat(), ) self.captures.append({"request_b64": request.to_b64(), "response_b64": None}) def capture_response(self, response: requests.Response) -> None: """ Add response data to the most recent capture entry. Args: response: The HTTP response object """ if not self.captures: return try: if hasattr(response, "json") and "application/json" in response.headers.get("Content-Type", ""): response_body = response.json() else: response_body = response.text if hasattr(response, "text") else "" except Exception: response_body = "" http_response = HTTPResponse( status_code=response.status_code, headers=dict(response.headers), body=response_body, timestamp=datetime.now(timezone.utc).isoformat(), ) self.captures[-1]["response_b64"] = http_response.to_b64() def save(self) -> None: """Save all HTTP captures to a JSON file.""" if not self.captures: return try: FileLock.run_with_lock(self.output_file, self._flush_traces_to_file) except Exception as e: logger.warning(f"Failed to save HTTP trace: {e}") def _flush_traces_to_file(self) -> None: """Flush captured traces to the output file (called within lock).""" output_path = Path(self.output_file) existing_traces: list[dict] = [] if output_path.exists() and output_path.stat().st_size > 0: with output_path.open("r") as f: existing_data = json.load(f) existing_traces = existing_data.get("traces", []) for capture in self.captures: request_b64 = capture.get("request_b64", "") response_b64 = capture.get("response_b64", "") request_data = None response_data = None if request_b64: request_data = json.loads(base64.b64decode(request_b64).decode()) if response_b64: response_data = json.loads(base64.b64decode(response_b64).decode()) existing_traces.append({"request": request_data, "response": response_data}) existing_traces.sort(key=lambda x: x["request"].get("timestamp", "") if x.get("request") else "") output_data = { "description": "HTTP trace data from Fabric API interactions", "total_traces": len(existing_traces), "traces": existing_traces, } with output_path.open("w") as f: json.dump(output_data, f, indent=2) class HTTPTracerFactory: """Factory class for creating HTTP tracer instances.""" @staticmethod def create() -> HTTPTracer: """ Create an HTTP tracer based on environment configuration. Returns: FileTracer if tracing is enabled via environment variable, NoOpTracer otherwise. """ from fabric_cicd.constants import VALID_ENABLE_FLAGS trace_enabled = os.environ.get(EnvVar.HTTP_TRACE_ENABLED.value, "").lower() in VALID_ENABLE_FLAGS return FileTracer() if trace_enabled else NoOpTracer() ================================================ FILE: src/fabric_cicd/_common/_item.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions and classes to manage Item operations.""" import os from dataclasses import dataclass, field from pathlib import Path from typing import ClassVar from fabric_cicd._common._file import File @dataclass class Item: """A class to represent a single item.""" type: str name: str description: str guid: str logical_id: str = field(default="") path: Path = field(default_factory=Path) item_files: list = field(default_factory=list) folder_id: str = field(default="") folder_path: str = field(default="") IMMUTABLE_FIELDS: ClassVar[set] = {"type", "name", "description"} skip_publish: bool = field(default=False) def __setattr__(self, key: str, value: any) -> None: """ Override setattr for 'immutable' fields. Args: key: The attribute name. value: The attribute value. """ if key in self.IMMUTABLE_FIELDS and hasattr(self, key): msg = f"item {key} is immutable" raise AttributeError(msg) super().__setattr__(key, value) @property def relative_path(self) -> str: """Return the relative path of the file.""" return str(self.file_path.relative_to(self.item_path).as_posix()) def collect_item_files(self) -> None: """Collect all files in the item path.""" self.item_files = [] for root, _dirs, files in os.walk(self.path): for file in files: full_path = Path(root, file) self.item_files.append(File(self.path, full_path)) ================================================ FILE: src/fabric_cicd/_common/_logging.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Logging utilities for the fabric_cicd package.""" import inspect import logging import re import sys import traceback from logging import LogRecord from logging.handlers import RotatingFileHandler from pathlib import Path from typing import ClassVar, Optional, Union from fabric_cicd import constants from fabric_cicd._common import _exceptions from fabric_cicd._common._color import Fore, Style class CustomFormatter(logging.Formatter): LEVEL_COLORS: ClassVar[dict[str, str]] = { "DEBUG": Fore.BLACK, "INFO": Fore.WHITE + Style.BRIGHT, "WARNING": Fore.YELLOW, "ERROR": Fore.RED, "CRITICAL": Style.BRIGHT + Fore.RED, } def format(self, record: LogRecord) -> str: level_color = self.LEVEL_COLORS.get(record.levelname, "") level_name = { "WARNING": "warn", "DEBUG": "debug", "INFO": "info", "ERROR": "error", "CRITICAL": "crit", }.get(record.levelname, "unknown") level_name = f"{level_color}[{level_name}]" timestamp = f"{self.formatTime(record, self.datefmt)}" message = f"{record.getMessage()}{Style.RESET_ALL}" # indent if the message contains "->" if constants.INDENT in message: message = message.replace(constants.INDENT, "") full_message = f"{' ' * 8} {timestamp} - {message}" else: # Calculate visual length by removing ANSI escape codes ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") # Get visual length of level_name without ANSI codes visual_level_length = len(ansi_escape.sub("", level_name)) # Pad to 16 visual characters padding = " " * max(0, 8 - visual_level_length) full_message = f"{level_name}{padding} {timestamp} - {message}" return full_message class PackageFilter(logging.Filter): """ Filter that only allows records from the fabric_cicd package logs. Args: debug_only: If True, only allows DEBUG level records. If False, allows all levels. """ def __init__(self, debug_only: bool = False) -> None: super().__init__() self.debug_only = debug_only def filter(self, record: LogRecord) -> bool: is_fabric_cicd = record.name.startswith("fabric_cicd") if self.debug_only: return is_fabric_cicd and record.levelno == logging.DEBUG return is_fabric_cicd """Helper functions to configure logging and handle exceptions across the fabric_cicd package.""" _DEFAULT_LOG_FILENAME = "fabric_cicd.error.log" _DEFAULT_LOG_FILE_FORMATTER = "%(asctime)s - %(levelname)s - %(name)s - %(message)s" _FABRIC_CICD_HANDLER_ATTR = "_fabric_cicd_managed" _FABRIC_CICD_EXTERNAL_HANDLER_ATTR = "_fabric_cicd_external" def _cleanup_external_handler_filters(root_logger: logging.Logger) -> None: """Remove PackageFilter from any external handler previously configured by fabric_cicd.""" for handler in list(root_logger.handlers): if getattr(handler, _FABRIC_CICD_EXTERNAL_HANDLER_ATTR, False): # Remove all PackageFilters that were added for f in list(handler.filters): if isinstance(f, PackageFilter): handler.removeFilter(f) # Remove the marker attribute delattr(handler, _FABRIC_CICD_EXTERNAL_HANDLER_ATTR) # Remove handler from root logger (don't close it - caller owns it) root_logger.removeHandler(handler) def _cleanup_managed_handlers(*loggers: logging.Logger) -> None: """Close and remove only handlers previously added by fabric_cicd.""" for logger_instance in loggers: # First, clean up any external handlers configured (filters only, don't close) _cleanup_external_handler_filters(logger_instance) # Then clean up fabric_cicd-managed handlers (close and remove) for handler in list(logger_instance.handlers): if getattr(handler, _FABRIC_CICD_HANDLER_ATTR, False): handler.close() logger_instance.removeHandler(handler) def _mark_handler(handler: logging.Handler) -> logging.Handler: """Mark a handler as managed by fabric_cicd.""" setattr(handler, _FABRIC_CICD_HANDLER_ATTR, True) return handler def _mark_external_handler(handler: logging.Handler) -> logging.Handler: """Mark an external handler as configured by fabric_cicd (for filter cleanup only).""" setattr(handler, _FABRIC_CICD_EXTERNAL_HANDLER_ATTR, True) return handler def _configure_default_file_handler() -> logging.Handler: """Configure the default file handler for standalone fabric_cicd usage.""" handler = logging.FileHandler( filename=_DEFAULT_LOG_FILENAME, mode="w", delay=True, ) handler.setFormatter(logging.Formatter(_DEFAULT_LOG_FILE_FORMATTER)) handler.addFilter(PackageFilter()) # All levels from fabric_cicd package logs return _mark_handler(handler) def _configure_external_file_handler( external_handler: Union[logging.FileHandler, RotatingFileHandler], level: int, debug_only_file: bool, ) -> logging.Handler: """ Configure an external file handler for fabric_cicd package logs. Reuses the external handler directly to preserve rotation behavior (if any). The external handler is not marked as fabric_cicd-managed (won't be closed), but is marked as external so filters can be cleaned up on reconfiguration. Note: Any existing PackageFilter is already removed by _cleanup_managed_handlers() before this function is called. """ # Add the appropriate filter if level == logging.DEBUG and debug_only_file: external_handler.addFilter(PackageFilter(debug_only=True)) else: external_handler.addFilter(PackageFilter()) # Mark as external in order to clean up filters later (but don't close it) return _mark_external_handler(external_handler) def _configure_console_handler(level: int) -> logging.StreamHandler: """Configure a console handler with the standard fabric_cicd formatter.""" handler = logging.StreamHandler() handler.setLevel(level) handler.setFormatter( CustomFormatter( "[%(levelname)s] %(asctime)s - %(message)s", datefmt="%H:%M:%S", ) ) return _mark_handler(handler) def _build_console_message(exception: BaseException, file_handler: Optional[logging.FileHandler] = None) -> str: """Build the user-facing console error message, optionally referencing the log file.""" # Write exception to console when file logging is disabled or when using an external file handler if file_handler is None or Path(file_handler.baseFilename).name != _DEFAULT_LOG_FILENAME: return f"{exception!s}" # Only reference the default log file which contains full error details log_file_path = Path(file_handler.baseFilename).resolve() return f"{exception!s}\n\nSee {log_file_path} for full details." def _build_file_message(exception: BaseException) -> str: """Build the log file message, including additional info if available.""" additional_info = getattr(exception, "additional_info", None) if additional_info is not None: return f"%s\n\nAdditional Info: \n{additional_info}" return "%s" """Main logging configuration and exception handling functions for fabric_cicd.""" def get_file_handler( logger: Optional[logging.Logger] = None, ) -> Optional[Union[logging.FileHandler, RotatingFileHandler]]: """ Get a file handler from a logger. Args: logger: The logger to search for a file handler. If None, searches the root logger for fabric_cicd-managed handlers only. Returns: The first FileHandler or RotatingFileHandler found, or None if not found. """ target_logger = logger if logger is not None else logging.getLogger() # When searching root logger, only return fabric_cicd-managed handlers check_managed = logger is None return next( ( h for h in target_logger.handlers if isinstance(h, (logging.FileHandler, RotatingFileHandler)) and (getattr(h, _FABRIC_CICD_HANDLER_ATTR, False) if check_managed else True) ), None, ) def configure_logger( level: int = logging.INFO, suppress_debug_console: bool = False, debug_only_file: bool = False, disable_log_file: bool = False, external_file_handler: Optional[Union[logging.FileHandler, RotatingFileHandler]] = None, ) -> None: """ Configure the logger. Args: level: The log level to set. Must be one of the standard logging levels. suppress_debug_console: Suppress DEBUG output to console (only applies when level is DEBUG). debug_only_file: Only write DEBUG messages to file (only applies when level is DEBUG). disable_log_file: Disable file logging entirely. external_file_handler: External file handler to append logs to instead of creating the default one. """ # Determine console level - suppress DEBUG to console if specified, otherwise same as level console_level = logging.INFO if suppress_debug_console and level == logging.DEBUG else level # Get all loggers root_logger = logging.getLogger() package_logger = logging.getLogger("fabric_cicd") console_only_logger = logging.getLogger("console_only") # Close and remove old handlers before adding new ones # This also cleans up any PackageFilter added to external handlers _cleanup_managed_handlers(root_logger, package_logger, console_only_logger) # Root logger - receives propagated records from fabric_cicd loggers # Holds the file handler so all fabric_cicd.* child loggers write to file via propagation # Set root logger level - for non-fabric_cicd packages: INFO if DEBUG, else ERROR root_logger.setLevel(level=logging.INFO if level == logging.DEBUG else logging.ERROR) # Configure file handler based on parameters if external_file_handler is not None: # Use provided external file handler for fabric_cicd logs root_logger.addHandler(_configure_external_file_handler(external_file_handler, level, debug_only_file)) elif not disable_log_file: # Use the default file handler for fabric_cicd logs root_logger.addHandler(_configure_default_file_handler()) # Package logger - primary logger for all fabric_cicd library logging # Writes to console via its own handler and to file via propagation to root package_logger.setLevel(level) package_logger.addHandler(_configure_console_handler(console_level)) # Console-only logger - used exclusively by exception_handler() to display # user-facing error messages on the terminal without writing them to the log file console_only_logger.setLevel(console_level) console_only_logger.addHandler(_configure_console_handler(console_level)) console_only_logger.propagate = False def exception_handler(exception_type: type[BaseException], exception: BaseException, traceback: traceback) -> None: """ Handle exceptions that are instances of any class from the _common._exceptions module. Args: exception_type: The type of the exception. exception: The exception instance. traceback: The traceback object. """ # Get all exception classes from the _common._exceptions module exception_classes = [cls for _, cls in inspect.getmembers(_exceptions, inspect.isclass)] # If the exception is not from _common._exceptions, use the default exception handler if not any(isinstance(exception, cls) for cls in exception_classes): sys.__excepthook__(exception_type, exception, traceback) return # Step 1: Write user-facing error message to console only (no file) file_handler = get_file_handler() # searches root logger for managed handlers console_message = _build_console_message(exception, file_handler) logging.getLogger("console_only").error(console_message) # Step 2: Write full stack trace to file only (not terminal) # Only write to file if using fabric_cicd default file handler (includes ERROR level logs) is_default_file_handler = file_handler is not None and Path(file_handler.baseFilename).name == _DEFAULT_LOG_FILENAME if is_default_file_handler: # Remove console handler from package logger so stack trace doesn't print to terminal package_logger = logging.getLogger("fabric_cicd") _cleanup_managed_handlers(package_logger) file_message = _build_file_message(exception) exception.logger.exception(file_message, exception, exc_info=(exception_type, exception, traceback)) def log_header(logger: logging.Logger, message: str) -> None: """ Logs a header message with a decorative line above and below it. Args: logger: The logger to use for logging the header message. message: The header message to log. """ line_separator = "#" * 100 formatted_message = f"########## {message}" formatted_message = f"{formatted_message} {line_separator[len(formatted_message) + 1 :]}" logger.info("") # Log a blank line before the header logger.info(f"{Fore.GREEN}{Style.BRIGHT}{line_separator}{Style.RESET_ALL}") logger.info(f"{Fore.GREEN}{Style.BRIGHT}{formatted_message}{Style.RESET_ALL}") logger.info(f"{Fore.GREEN}{Style.BRIGHT}{line_separator}{Style.RESET_ALL}") logger.info("") ================================================ FILE: src/fabric_cicd/_common/_validate_env_vars.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions for validating environment variables and constants used by fabric-cicd.""" import logging import os import re from urllib.parse import urlsplit from fabric_cicd._common._exceptions import InputError logger = logging.getLogger(__name__) # Define a regular expression for valid hostnames # Matches: any subdomain of []api.fabric.microsoft.com or []api.powerbi.com _VALID_HOSTNAME_REGEX = re.compile(r"^([\w-]+\.)*[\w-]*api\.(fabric\.microsoft\.com|powerbi\.com)\Z", re.IGNORECASE) # Regular expression for valid GUIDs with dashes VALID_GUID_REGEX = r"^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$" # Constants that hold API URLs and require URL validation _URL_CONSTANTS = {"DEFAULT_API_ROOT_URL", "FABRIC_API_ROOT_URL"} def validate_api_url(url: str, label: str) -> str: """ Validates an API URL string. Validates the value is non-empty, the scheme is https, the hostname matches allowed patterns, and no path components are present. Args: url: The URL string to validate. label: A human-readable label for error messages (e.g., env var name or config key). Returns: str: The validated URL with trailing slashes removed. """ if not url.strip(): msg = f"'{label}' must resolve to a non-empty string." raise InputError(msg, logger) # Parse the URL using urlsplit parsed = urlsplit(url) if parsed.scheme != "https": msg = f"Invalid or missing scheme in '{label}': '{url}'. URL must use HTTPS scheme." raise InputError(msg, logger) hostname = parsed.hostname or "" if not _VALID_HOSTNAME_REGEX.match(hostname): msg = f"'{label}' has invalid hostname: {hostname}" raise InputError(msg, logger) if parsed.path and parsed.path not in ("", "/"): msg = f"'{label}' should be a root URL without path components. Got path: '{parsed.path}'" raise InputError(msg, logger) return url.rstrip("/") def validate_env_var_api_url(env_var_name: str, default_value: str) -> str: """ Validates and returns the API URL from an environment variable. Validates the scheme is https and the hostname matches allowed patterns. Args: env_var_name: Name of the environment variable default_value: Default value if environment variable is not set (full URL with https://) Returns: str: The original validated API URL value, or the default if env var is not set. """ value = os.environ.get(env_var_name, default_value) return validate_api_url(value, f"environment variable '{env_var_name}'") def _get_fabric_fqdn_url(workspace_id: str) -> str: """ Transform workspace ID to FQDN format for private-link-enabled Fabric workspaces. Args: workspace_id: The workspace ID string in standard GUID format with dashes (e.g., "f953f3da-c5f0-4e36-a644-c85933e35e2f"). Returns: The FQDN URL string in the format: https://.z.w.api.fabric.microsoft.com Examples: >>> url = _get_fabric_fqdn_url("f953f3da-c5f0-4e36-a644-c85933e35e2f") >>> url 'https://f953f3dac5f04e36a644c85933e35e2f.zf9.w.api.fabric.microsoft.com' """ if not re.match(VALID_GUID_REGEX, workspace_id): msg = f"workspace_id must be a valid GUID with dashes, got: '{workspace_id}'" raise ValueError(msg) no_dashes = workspace_id.replace("-", "") first_two = no_dashes[:2] return f"https://{no_dashes}.z{first_two}.w.api.fabric.microsoft.com" ================================================ FILE: src/fabric_cicd/_common/_validate_input.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """ Following functions are leveraged to validate user input for the fabric-cicd package Primarily used for the FabricWorkspace class, but also intended to be leveraged for any user input throughout the package """ import logging import re from pathlib import Path from typing import Optional from azure.core.credentials import TokenCredential import fabric_cicd.constants as constants from fabric_cicd._common._exceptions import InputError from fabric_cicd.constants import FeatureFlag, OperationType from fabric_cicd.fabric_workspace import FabricWorkspace logger = logging.getLogger(__name__) def validate_data_type(expected_type: str, variable_name: str, input_value: any) -> any: """ Validate the data type of the input value. Args: expected_type: The expected data type. variable_name: The name of the variable. input_value: The input value to validate. """ # Mapping of expected types to their validation functions type_validators = { "string": lambda x: isinstance(x, str), "bool": lambda x: isinstance(x, bool), "list": lambda x: isinstance(x, list), "list[string]": lambda x: isinstance(x, list) and all(isinstance(item, str) for item in x), "FabricWorkspace": lambda x: isinstance(x, FabricWorkspace), "TokenCredential": lambda x: isinstance(x, TokenCredential), } # Check if the expected type is valid and if the input matches the expected type if expected_type not in type_validators or not type_validators[expected_type](input_value): msg = f"The provided {variable_name} is not of type {expected_type}." raise InputError(msg, logger) return input_value def validate_item_type_in_scope(input_value: Optional[list]) -> list: """ Validate the item type in scope. Args: input_value: The input value to validate. If None, defaults to all supported item types. """ accepted_item_types = constants.ACCEPTED_ITEM_TYPES # If None, return all accepted item types if input_value is None: return list(accepted_item_types) validate_data_type("list[string]", "item_type_in_scope", input_value) for item_type in input_value: if item_type not in accepted_item_types: msg = f"Invalid or unsupported item type: '{item_type}'. Must be one of {', '.join(accepted_item_types)}." raise InputError(msg, logger) return input_value def validate_repository_directory(input_value: str) -> Path: """ Validate the repository directory and convert string to Path object Args: input_value: The input value to validate. """ validate_data_type("string", "repository_directory", input_value) directory = Path(input_value) if not directory.is_dir(): msg = f"The provided repository_directory '{input_value}' does not exist." raise InputError(msg, logger) if not directory.is_absolute(): absolute_directory = directory.resolve() logger.info(f"Relative directory path '{directory}' resolved as '{absolute_directory}'") directory = absolute_directory return directory def validate_workspace_id(input_value: str) -> str: """ Validate the workspace ID. Args: input_value: The input value to validate. """ validate_data_type("string", "workspace_id", input_value) if not re.match(constants.VALID_GUID_REGEX, input_value): msg = "The provided workspace_id is not a valid guid." raise InputError(msg, logger) return input_value def validate_workspace_name(input_value: str) -> str: """ Validate the workspace name. Args: input_value: The input value to validate. """ validate_data_type("string", "workspace_name", input_value) return input_value def validate_environment(input_value: str) -> str: """ Validate the environment. Args: input_value: The input value to validate. """ validate_data_type("string", "environment", input_value) return input_value def validate_fabric_workspace_obj(input_value: FabricWorkspace) -> FabricWorkspace: """ Validate the FabricWorkspace object. Args: input_value: The input value to validate. """ validate_data_type("FabricWorkspace", "fabric_workspace_obj", input_value) return input_value def validate_token_credential(input_value: TokenCredential) -> TokenCredential: """ Validate the token credential. Args: input_value: The input value to validate. """ validate_data_type("TokenCredential", "credential", input_value) return input_value def validate_experimental_param( param_value: Optional[str], required_flag: "FeatureFlag", warning_message: str, risk_warning: str, ) -> None: """ Generic validation for optional parameters requiring experimental feature flags. Args: param_value: The parameter value (None means skip validation). required_flag: The specific feature flag required (in addition to experimental). warning_message: Primary warning message when feature is enabled. risk_warning: Risk/caution warning message. Raises: InputError: If required feature flags are not enabled. """ if param_value is None: return if ( FeatureFlag.ENABLE_EXPERIMENTAL_FEATURES.value not in constants.FEATURE_FLAG or required_flag.value not in constants.FEATURE_FLAG ): msg = f"Feature flags 'enable_experimental_features' and '{required_flag.value}' must be set." raise InputError(msg, logger) logger.warning(warning_message) logger.warning(risk_warning) def validate_items_to_include(items_to_include: Optional[list[str]], operation: "OperationType") -> None: """ Validate items_to_include parameter and check required feature flags. Args: items_to_include: List of items in "item_name.item_type" format, or None. operation: The type of operation being performed (publish or unpublish). Raises: InputError: If required feature flags are not enabled. """ validate_experimental_param( param_value=items_to_include, required_flag=FeatureFlag.ENABLE_ITEMS_TO_INCLUDE, warning_message=f"Selective {operation.value} is enabled.", risk_warning=f"Using items_to_include is risky as it can prevent needed dependencies from being {operation.value}. Use at your own risk.", ) def validate_folder_path_exclude_regex(folder_path_exclude_regex: Optional[str]) -> None: """ Validate folder_path_exclude_regex parameter and check required feature flags. Args: folder_path_exclude_regex: Regex pattern to exclude items based on their folder path, or None. Raises: InputError: If required feature flags are not enabled. """ validate_experimental_param( param_value=folder_path_exclude_regex, required_flag=FeatureFlag.ENABLE_EXCLUDE_FOLDER, warning_message="Folder path exclusion is enabled.", risk_warning="Using folder_path_exclude_regex is risky as it can prevent needed dependencies from being deployed. Use at your own risk.", ) if not isinstance(folder_path_exclude_regex, str): msg = "folder_path_exclude_regex must be a string." raise InputError(msg, logger) if folder_path_exclude_regex == "": msg = "folder_path_exclude_regex must not be an empty string. Provide a valid regex pattern or omit the parameter." raise InputError(msg, logger) def validate_folder_path_to_include(folder_path_to_include: Optional[list[str]]) -> None: """ Validate folder_path_to_include parameter and check required feature flags. Args: folder_path_to_include: List of folder paths with format ["/folder1", "/folder2", ...], or None. Raises: InputError: If required feature flags are not enabled. """ validate_experimental_param( param_value=folder_path_to_include, required_flag=FeatureFlag.ENABLE_INCLUDE_FOLDER, warning_message="Folder path inclusion is enabled.", risk_warning="Using folder_path_to_include is risky as it can prevent needed dependencies from being deployed. Use at your own risk.", ) if not isinstance(folder_path_to_include, list): msg = "folder_path_to_include must be a list of folder paths." raise InputError(msg, logger) if not folder_path_to_include: msg = "folder_path_to_include must not be an empty list. Provide folder paths or omit the parameter." raise InputError(msg, logger) def validate_shortcut_exclude_regex(shortcut_exclude_regex: Optional[str]) -> None: """ Validate shortcut_exclude_regex parameter and check required feature flags. Args: shortcut_exclude_regex: Regex pattern to exclude specific shortcuts from being published, or None. Raises: InputError: If required feature flags are not enabled. """ validate_experimental_param( param_value=shortcut_exclude_regex, required_flag=FeatureFlag.ENABLE_SHORTCUT_EXCLUDE, warning_message="Shortcut exclusion is enabled.", risk_warning="Using shortcut_exclude_regex will selectively exclude shortcuts from being deployed to lakehouses. Use with caution.", ) def validate_git_compare_ref(git_compare_ref: str) -> str: """ Validate the git_compare_ref parameter to prevent git flag injection. Args: git_compare_ref: The git ref to compare against. Raises: InputError: If the ref is empty, starts with '-', or contains invalid characters. """ validate_data_type("string", "git_compare_ref", git_compare_ref) if not git_compare_ref.strip(): msg = "git_compare_ref must not be an empty string." raise InputError(msg, logger) if git_compare_ref.startswith("-"): msg = "git_compare_ref must not start with '-' to prevent git flag injection." raise InputError(msg, logger) # Allow only characters valid in git refs: alphanumeric, /, ., ~, ^, -, _ if not re.match(r"^[a-zA-Z0-9/_.\-~^@{}]+$", git_compare_ref): msg = f"git_compare_ref '{git_compare_ref}' contains invalid characters." raise InputError(msg, logger) return git_compare_ref ================================================ FILE: src/fabric_cicd/_items/__init__.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. from fabric_cicd._common._exceptions import PublishError from fabric_cicd._items._base_publisher import ItemPublisher, ParallelConfig __all__ = [ "ItemPublisher", "ParallelConfig", "PublishError", ] ================================================ FILE: src/fabric_cicd/_items/_activator.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions to process and deploy Reflex item.""" from fabric_cicd._items._base_publisher import ItemPublisher from fabric_cicd.constants import ItemType class ActivatorPublisher(ItemPublisher): """Publisher for Reflex AKA Activator items.""" item_type = ItemType.REFLEX.value ================================================ FILE: src/fabric_cicd/_items/_apacheairflowjob.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions to process and deploy Apache Airflow Job item.""" from fabric_cicd._items._base_publisher import ItemPublisher from fabric_cicd.constants import ItemType class ApacheAirflowJobPublisher(ItemPublisher): """Publisher for Apache Airflow Job items.""" item_type = ItemType.APACHE_AIRFLOW_JOB.value ================================================ FILE: src/fabric_cicd/_items/_base_publisher.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Base interface for all item publishers.""" import logging from abc import ABC, abstractmethod from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass from typing import Callable, Optional from fabric_cicd._common._exceptions import PublishError from fabric_cicd._common._item import Item from fabric_cicd.constants import PARALLEL_MAX_WORKERS, ItemType from fabric_cicd.fabric_workspace import FabricWorkspace logger = logging.getLogger(__name__) @dataclass class ParallelConfig: """Configuration for parallel execution behavior of a publisher. This dataclass controls how the base ItemPublisher.publish_all() method executes publish_one() calls - either in parallel or sequentially. Attributes: enabled: If True, publish_one calls run in parallel using ThreadPoolExecutor. If False, items are published sequentially. Default is True. max_workers: Maximum number of concurrent threads. None means use ThreadPoolExecutor default. ordered_items_func: Optional callable that returns an ordered list of item names. When provided, items are published sequentially in this order. This takes precedence over `enabled=True`. """ enabled: bool = True max_workers: Optional[int] = PARALLEL_MAX_WORKERS ordered_items_func: Optional[Callable[["ItemPublisher"], list[str]]] = None class Publisher(ABC): """Base interface for all publishers.""" def __init__(self, fabric_workspace_obj: "FabricWorkspace") -> None: """ Initialize the publisher with a FabricWorkspace object. Args: fabric_workspace_obj: The FabricWorkspace object containing items to be published. """ self.fabric_workspace_obj = fabric_workspace_obj @abstractmethod def publish_one(self, name: str, obj: object) -> None: """ Publish a single object. Args: name: The name of the object to publish. obj: The object to publish. """ raise NotImplementedError @abstractmethod def publish_all(self) -> None: """Publish all objects.""" raise NotImplementedError class ItemPublisher(Publisher): """ Base interface for all item type publishers. Provides a default parallel publish_all() implementation that: - Executes publish_one() calls in parallel using ThreadPoolExecutor - Aggregates errors from all failed items into a single PublishError - Supports pre/post hooks via pre_publish_all() and post_publish_all() - Can be configured via the parallel_config class attribute Subclasses can customize behavior by: - Setting parallel_config to control parallelization - Overriding pre_publish_all() for setup before publishing - Overriding post_publish_all() for cleanup after publishing - Overriding get_items_to_publish() to filter or order items - Overriding get_unpublish_order() for dependency-aware unpublishing - Overriding post_publish_all_check() for async publish state verification Publish Lifecycle: 1. pre_publish_all() 2. get_items_to_publish() 3. publish_one() - called for each item 4. post_publish_all() 5. post_publish_all_check() - if has_async_publish_check Unpublish Hook: - get_unpublish_order() - if has_dependency_tracking """ # region Class Attributes item_type: str """Mandatory property to be set by each publisher subclass""" parallel_config: ParallelConfig = ParallelConfig() """Configuration for parallel execution - subclasses can override with their own ParallelConfig""" has_async_publish_check: bool = False """Set to True if this publisher implements post_publish_all_check() for async state verification""" has_dependency_tracking: bool = False """Set to True if this publisher implements get_unpublish_order() for dependency ordering""" # endregion # region Initialization & Factory def __init__(self, fabric_workspace_obj: "FabricWorkspace") -> None: """ Initialize the publisher with a FabricWorkspace object. Args: fabric_workspace_obj: The FabricWorkspace object containing items to be published. """ super().__init__(fabric_workspace_obj) @staticmethod def create(item_type: ItemType, fabric_workspace_obj: "FabricWorkspace") -> "ItemPublisher": """ Factory method to create the appropriate publisher for a given item type. Args: item_type: The ItemType enum value for which to create a publisher. fabric_workspace_obj: The FabricWorkspace object containing items to be published. Returns: An instance of the appropriate ItemPublisher subclass. Raises: ValueError: If the item type is not supported. """ from fabric_cicd._items._activator import ActivatorPublisher from fabric_cicd._items._apacheairflowjob import ApacheAirflowJobPublisher from fabric_cicd._items._copyjob import CopyJobPublisher from fabric_cicd._items._dataagent import DataAgentPublisher from fabric_cicd._items._databuildtooljob import DataBuildToolJobPublisher from fabric_cicd._items._dataflowgen2 import DataflowPublisher from fabric_cicd._items._datapipeline import DataPipelinePublisher from fabric_cicd._items._environment import EnvironmentPublisher from fabric_cicd._items._eventhouse import EventhousePublisher from fabric_cicd._items._eventstream import EventstreamPublisher from fabric_cicd._items._graphqlapi import GraphQLApiPublisher from fabric_cicd._items._kqldashboard import KQLDashboardPublisher from fabric_cicd._items._kqldatabase import KQLDatabasePublisher from fabric_cicd._items._kqlqueryset import KQLQuerysetPublisher from fabric_cicd._items._lakehouse import LakehousePublisher from fabric_cicd._items._mirroreddatabase import MirroredDatabasePublisher from fabric_cicd._items._mlexperiment import MLExperimentPublisher from fabric_cicd._items._mounteddatafactory import MountedDataFactoryPublisher from fabric_cicd._items._notebook import NotebookPublisher from fabric_cicd._items._ontology import OntologyPublisher from fabric_cicd._items._report import ReportPublisher from fabric_cicd._items._semanticmodel import SemanticModelPublisher from fabric_cicd._items._sparkjobdefinition import SparkJobDefinitionPublisher from fabric_cicd._items._sqldatabase import SQLDatabasePublisher from fabric_cicd._items._userdatafunction import UserDataFunctionPublisher from fabric_cicd._items._variablelibrary import VariableLibraryPublisher from fabric_cicd._items._warehouse import WarehousePublisher publisher_mapping = { ItemType.VARIABLE_LIBRARY: VariableLibraryPublisher, ItemType.WAREHOUSE: WarehousePublisher, ItemType.MIRRORED_DATABASE: MirroredDatabasePublisher, ItemType.LAKEHOUSE: LakehousePublisher, ItemType.SQL_DATABASE: SQLDatabasePublisher, ItemType.ENVIRONMENT: EnvironmentPublisher, ItemType.USER_DATA_FUNCTION: UserDataFunctionPublisher, ItemType.EVENTHOUSE: EventhousePublisher, ItemType.SPARK_JOB_DEFINITION: SparkJobDefinitionPublisher, ItemType.NOTEBOOK: NotebookPublisher, ItemType.SEMANTIC_MODEL: SemanticModelPublisher, ItemType.REPORT: ReportPublisher, ItemType.COPY_JOB: CopyJobPublisher, ItemType.DATA_BUILD_TOOL_JOB: DataBuildToolJobPublisher, ItemType.KQL_DATABASE: KQLDatabasePublisher, ItemType.KQL_QUERYSET: KQLQuerysetPublisher, ItemType.REFLEX: ActivatorPublisher, ItemType.EVENTSTREAM: EventstreamPublisher, ItemType.KQL_DASHBOARD: KQLDashboardPublisher, ItemType.DATAFLOW: DataflowPublisher, ItemType.DATA_PIPELINE: DataPipelinePublisher, ItemType.GRAPHQL_API: GraphQLApiPublisher, ItemType.APACHE_AIRFLOW_JOB: ApacheAirflowJobPublisher, ItemType.MOUNTED_DATA_FACTORY: MountedDataFactoryPublisher, ItemType.DATA_AGENT: DataAgentPublisher, ItemType.ML_EXPERIMENT: MLExperimentPublisher, ItemType.ONTOLOGY: OntologyPublisher, } publisher_class = publisher_mapping.get(item_type) if publisher_class is None: msg = f"No publisher found for item type: {item_type}" raise ValueError(msg) return publisher_class(fabric_workspace_obj) @staticmethod def get_item_types_to_publish(fabric_workspace_obj: "FabricWorkspace") -> list[tuple[int, ItemType]]: """ Get the ordered list of item types that should be published. Returns item types that are both in scope and have items in the repository, ordered according to SERIAL_ITEM_PUBLISH_ORDER. Args: fabric_workspace_obj: The FabricWorkspace object containing scope and repository info. Returns: List of (order_num, ItemType) tuples for item types that should be published. """ from fabric_cicd import constants result = [] for order_num, item_type in constants.SERIAL_ITEM_PUBLISH_ORDER.items(): if ( item_type.value in fabric_workspace_obj.item_type_in_scope and item_type.value in fabric_workspace_obj.repository_items ): result.append((order_num, item_type)) return result @staticmethod def get_item_types_to_unpublish(fabric_workspace_obj: "FabricWorkspace") -> list[str]: """ Get the ordered list of item types that should be unpublished. Returns item types in reverse publish order that are in scope, have deployed items, and meet feature flag requirements. Logs warnings for skipped item types. Args: fabric_workspace_obj: The FabricWorkspace object containing scope and deployed items info. Returns: List of item type strings in the order they should be unpublished. """ from fabric_cicd import constants unpublish_order = [] for item_type in reversed(list(constants.SERIAL_ITEM_PUBLISH_ORDER.values())): if ( item_type.value in fabric_workspace_obj.item_type_in_scope and item_type.value in fabric_workspace_obj.deployed_items ): unpublish_flag = constants.UNPUBLISH_FLAG_MAPPING.get(item_type.value) # Append item_type if no feature flag is required or the corresponding flag is enabled if not unpublish_flag or unpublish_flag in constants.FEATURE_FLAG: unpublish_order.append(item_type.value) elif unpublish_flag and unpublish_flag not in constants.FEATURE_FLAG: # Log warning when unpublish is skipped due to missing feature flag logger.warning( f"Skipping unpublish for {item_type.value} items because the '{unpublish_flag}' feature flag is not enabled." ) return unpublish_order @staticmethod def get_orphaned_items( fabric_workspace_obj: "FabricWorkspace", item_type: str, item_name_exclude_regex: Optional[str] = None, items_to_include: Optional[list[str]] = None, ) -> list[str]: """ Get the list of orphaned items that should be unpublished for a given item type. Orphaned items are those deployed but not present in the repository, filtered by exclusion regex or items_to_include list. Args: fabric_workspace_obj: The FabricWorkspace object containing deployed and repository items. item_type: The item type string to check for orphans. item_name_exclude_regex: Optional regex pattern to exclude items from unpublishing. items_to_include: Optional list of items in "name.type" format to include for unpublishing. Returns: List of item names that should be unpublished. """ import re deployed_names = set(fabric_workspace_obj.deployed_items.get(item_type, {}).keys()) repository_names = set(fabric_workspace_obj.repository_items.get(item_type, {}).keys()) to_delete_set = deployed_names - repository_names if items_to_include is not None: # Filter to only items in the include list return [name for name in to_delete_set if f"{name}.{item_type}" in items_to_include] if item_name_exclude_regex: # Filter out items matching the exclude regex regex_pattern = re.compile(item_name_exclude_regex) return [name for name in to_delete_set if not regex_pattern.match(name)] return list(to_delete_set) # endregion # region Public Methods def publish_all(self) -> None: """ Execute the publish operation for this item type. 1. Calls pre_publish_all() for any setup operations 2. Gets items via get_items_to_publish() 3. Publishes items (parallel or sequential based on parallel_config) 4. Calls post_publish_all() for any finalization 5. Raises PublishError if any items failed The parallel_config class attribute controls execution: - If ordered_items_func is set: publishes in that order sequentially - If enabled=True: publishes in parallel - If enabled=False: publishes sequentially Raises: PublishError: If one or more items failed to publish. """ self.pre_publish_all() all_items = self.fabric_workspace_obj.repository_items.get(self.item_type, {}) items = self.get_items_to_publish() # Mark excluded items as skip_publish=True so post_publish_all() hooks # can reliably skip them. Included items are left unchanged (False). skipped = [name for name in all_items if name not in items] if skipped: logger.info(f"Skipping {self.item_type} item(s) due to items_to_include filter: {skipped}") for name in skipped: all_items[name].skip_publish = True if not items: self.post_publish_all() return config = getattr(self.__class__, "parallel_config", ParallelConfig()) if config.ordered_items_func is not None: order = config.ordered_items_func(self) errors = self._publish_items_ordered(items, order) elif config.enabled: errors = self._publish_items_parallel(items) else: errors = self._publish_items_sequential(items) self.post_publish_all() if errors: raise PublishError(errors, logger) def publish_one(self, item_name: str, _item: "Item") -> None: """ Publish a single item. Args: item_name: The name of the item to publish. _item: The Item object to publish. Default implementation publishes the item using _publish_item. Subclasses can override this method for custom publishing logic. """ self.fabric_workspace_obj._publish_item(item_name=item_name, item_type=self.item_type) def get_items_to_publish(self) -> dict[str, "Item"]: """ Get the items to publish for this item type. Returns: Dictionary mapping item names to Item objects, pre-filtered by items_to_include when set so that only relevant items are iterated. Subclasses can override to filter or transform the items. Note: The base implementation applies ``FabricWorkspace.items_to_include`` filtering. To override this method and preserve this behavior, call ``super().get_items_to_publish()`` to keep ``items_to_include`` support, then apply any additional selection logic. Items NOT returned by this method will have ``skip_publish=True`` set on them by ``publish_all()`` before ``post_publish_all()`` is called. This ensures that ``post_publish_all()`` hooks (e.g. shortcut publishing, connection binding) can reliably use ``item.skip_publish`` to determine whether an item was published. """ all_items = self.fabric_workspace_obj.repository_items.get(self.item_type, {}) items_to_include = self.fabric_workspace_obj.items_to_include if not items_to_include: return all_items normalized_include_set = {i.lower() for i in items_to_include} return { name: item for name, item in all_items.items() if f"{name}.{self.item_type}".lower() in normalized_include_set } def get_unpublish_order(self, items_to_unpublish: list[str]) -> list[str]: """ Get the ordered list of item names based on dependencies for unpublishing. Args: items_to_unpublish: List of item names to be unpublished. Returns: List of item names in the order they should be unpublished (reverse dependency order). Default implementation returns items in their original order. Subclasses with dependency tracking should override for proper ordering. """ return items_to_unpublish def pre_publish_all(self) -> None: """ Hook called before publishing any items. Subclasses can override to perform setup, validation, or refresh operations. Default implementation does nothing. """ pass def post_publish_all(self) -> None: """ Hook called after all items have been published successfully. Subclasses can override to perform cleanup, binding, or finalization. Default implementation does nothing. """ pass def post_publish_all_check(self) -> None: """ Hook called after publish_all completes to verify async publish state. Subclasses can override this to check the state of asynchronous publish operations (e.g., Environment items that have async publish workflows). Default implementation does nothing. This method is called separately from publish_all() and should be invoked by the orchestration layer after all items of this type have been published. """ pass # endregion # region Publishing def _publish_items_parallel(self, items: dict[str, "Item"]) -> list[tuple[str, Exception]]: """ Publish items in parallel using ThreadPoolExecutor. Args: items: Dictionary mapping item names to Item objects. Returns: List of (item_name, exception) tuples for failed items. """ errors: list[tuple[str, Exception]] = [] config = getattr(self.__class__, "parallel_config", ParallelConfig()) with ThreadPoolExecutor(max_workers=config.max_workers) as executor: futures = { executor.submit(self.publish_one, item_name, item): (item_name, item) for item_name, item in items.items() } for future in as_completed(futures): item_name, _ = futures[future] try: future.result() except Exception as e: logger.error(f"Failed to publish {self.item_type} '{item_name}': {e}") errors.append((item_name, e)) return errors def _publish_items_sequential(self, items: dict[str, "Item"]) -> list[tuple[str, Exception]]: """ Publish items sequentially. Args: items: Dictionary mapping item names to Item objects. Returns: List of (item_name, exception) tuples for failed items. """ errors: list[tuple[str, Exception]] = [] for item_name, item in items.items(): try: self.publish_one(item_name, item) except Exception as e: logger.error(f"Failed to publish {self.item_type} '{item_name}': {e}") errors.append((item_name, e)) return errors def _publish_items_ordered(self, items: dict[str, "Item"], order: list[str]) -> list[tuple[str, Exception]]: """ Publish items in a specific order sequentially. Args: items: Dictionary mapping item names to Item objects. order: List of item names in the order they should be published. Returns: List of (item_name, exception) tuples for failed items. """ errors: list[tuple[str, Exception]] = [] for item_name in order: if item_name in items: item = items[item_name] try: self.publish_one(item_name, item) except Exception as e: logger.error(f"Failed to publish {self.item_type} '{item_name}': {e}") errors.append((item_name, e)) return errors # endregion ================================================ FILE: src/fabric_cicd/_items/_copyjob.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions to process and deploy Copy Job item.""" from fabric_cicd._items._base_publisher import ItemPublisher from fabric_cicd.constants import ItemType class CopyJobPublisher(ItemPublisher): """Publisher for Copy Job items.""" item_type = ItemType.COPY_JOB.value ================================================ FILE: src/fabric_cicd/_items/_dataagent.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions to process and deploy Data Agent item.""" import logging from fabric_cicd._common._item import Item from fabric_cicd._items._base_publisher import ItemPublisher from fabric_cicd.constants import EXCLUDE_PATH_REGEX_MAPPING, ItemType logger = logging.getLogger(__name__) class DataAgentPublisher(ItemPublisher): """Publisher for Data Agent items.""" item_type = ItemType.DATA_AGENT.value def publish_one(self, item_name: str, _item: Item) -> None: """Publish a single Data Agent item.""" self.fabric_workspace_obj._publish_item( item_name=item_name, item_type=self.item_type, exclude_path=EXCLUDE_PATH_REGEX_MAPPING.get(self.item_type) ) ================================================ FILE: src/fabric_cicd/_items/_databuildtooljob.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions to process and deploy Data Build Tool Job item.""" from fabric_cicd._items._base_publisher import ItemPublisher from fabric_cicd.constants import ItemType class DataBuildToolJobPublisher(ItemPublisher): """Publisher for Data Build Tool Job items.""" item_type = ItemType.DATA_BUILD_TOOL_JOB.value ================================================ FILE: src/fabric_cicd/_items/_dataflowgen2.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions to process and deploy Dataflow Gen2 item.""" import logging import re from fabric_cicd import FabricWorkspace, constants from fabric_cicd._common._exceptions import ParsingError from fabric_cicd._common._file import File from fabric_cicd._common._item import Item from fabric_cicd._items._base_publisher import ItemPublisher, ParallelConfig from fabric_cicd._parameter._utils import ( check_replacement, extract_find_value, extract_parameter_filters, extract_replace_value, ) from fabric_cicd.constants import ItemType logger = logging.getLogger(__name__) def set_dataflow_publish_order(workspace_obj: FabricWorkspace, item_type: str) -> list[str]: """ Sets the publish order where the source dataflow, if present always proceeds the referencing dataflow. This only applies to dataflows that reference another dataflow in the repository and the referenced dataflowId is parameterized correctly (using the items attribute variable). Algorithm for determining dataflow publish order: 1. Find all dataflows that reference other dataflows in the repository 2. Build a dependency graph where each dataflow depends on its source dataflow 3. Use a modified depth-first search with cycle detection to create a topological sort ensuring that source dataflows are published before the dataflows that reference them 4. Add any remaining standalone dataflows (without dependencies) to the end of the publish order Args: workspace_obj: The FabricWorkspace object. item_type: Type of item (e.g., 'Dataflow'). """ publish_order = [] visited = set() temp_visited = set() # If the find_replace parameter doesn't exist, skip sorting dataflows by dependencies param_dict = workspace_obj.environment_parameter.get("find_replace", []) if not param_dict: logger.warning( "find_replace parameter not found - dataflows will not be checked for dependencies. " "Dataflows will be published in arbitrary order, which may cause errors in the deployed items" ) return list(workspace_obj.repository_items.get(item_type, {}).keys()) # Otherwise, collect dataflow items with a source dataflow for item in workspace_obj.repository_items.get(item_type, {}).values(): for file in item.item_files: # Check if a dataflow is referenced in the file if file.type == "text" and str(file.file_path).endswith(".pq") and contains_source_dataflow(file.contents): # Try to get info associated with the dataflow reference dataflow_name, dataflow_workspace_id, dataflow_id = get_source_dataflow_name( workspace_obj, file.contents, item.name, file.file_path ) # If the dataflow is found in the repository, add it to the dependencies dictionary if dataflow_name: workspace_obj.dataflow_dependencies[item.name] = { "source_name": dataflow_name, "source_workspace_id": dataflow_workspace_id, "source_id": dataflow_id, } else: logger.warning(f"The '{item.name}' dataflow will be published without considering its dependency") def add_dataflow_with_dependency(item: str) -> bool: """ Recursively adds an item and its dependency to the publish order. Returns True if successful, False if a cycle is detected. """ # Dataflow was already processed, no need to process again if item in visited: return True # If the item is already in the temporary visited set, it indicates a cycle if item in temp_visited: msg = f"Circular dependency found for item {item}. Cannot determine a valid publish order" raise ParsingError(msg, logger) # Add the item to the temporary visited set temp_visited.add(item) # First add the dependency if it exists if workspace_obj.dataflow_dependencies.get(item): dependency = workspace_obj.dataflow_dependencies[item]["source_name"] # Propagate cycle detection if not add_dataflow_with_dependency(dependency): return False # Then add the current item publish_order.append(item) visited.add(item) # Remove from temporary set temp_visited.remove(item) return True # Process each item in the dataflow dependencies for item in list(workspace_obj.dataflow_dependencies.keys()): add_dataflow_with_dependency(item) # Add any remaining dataflows from the repository that aren't in the publish order (standalone dataflows) for item_name in workspace_obj.repository_items.get(item_type, {}): if item_name not in visited: publish_order.append(item_name) return publish_order def contains_source_dataflow(file_content: str) -> bool: """A helper function to check if the file content contains a source dataflow reference.""" try: # Check if file contains the PowerPlatform.Dataflows pattern (group 1 of the regex) match = re.search(constants.DATAFLOW_SOURCE_REGEX, file_content, re.DOTALL) return match is not None and bool(match.group(1)) except (re.error, TypeError, IndexError) as e: logger.debug(f"Error checking for source dataflow: {e}") return False def get_source_dataflow_ids(file_content: str, item_name: str) -> tuple[str, str]: """A helper function to get the dataflow ID and workspace ID of a referenced dataflow.""" try: match = re.search(constants.DATAFLOW_SOURCE_REGEX, file_content, re.DOTALL) if not match: msg = f"No dataflow source pattern found in the '{item_name}' file content" raise ParsingError(msg, logger) # Extract the source dataflow IDs from the regex match dataflow_workspace_id = match.group(2) dataflow_id = match.group(3) except Exception as e: msg = f"Error extracting dataflow information from file content: {e}" raise ParsingError(msg, logger) from e # Validate the extracted IDs are valid GUIDs if not dataflow_workspace_id or not re.match(constants.VALID_GUID_REGEX, dataflow_workspace_id): msg = f"Invalid workspace ID: {dataflow_workspace_id} in '{item_name}' file content" raise ParsingError(msg, logger) if not dataflow_id or not re.match(constants.VALID_GUID_REGEX, dataflow_id): msg = f"Invalid dataflow ID: {dataflow_id} in '{item_name}' file content" raise ParsingError(msg, logger) return dataflow_workspace_id, dataflow_id def get_source_dataflow_name( workspace_obj: FabricWorkspace, file_content: str, item_name: str, file_path: str ) -> tuple[str, str, str]: """ A helper function to extract the dataflow name, dataflow workspaceId and dataflowId associated with the source dataflow referenced in the file content. The source dataflow name is obtained by using the matching parameter dictionary input, if present. """ # Get the IDs of the source dataflow dataflow_workspace_id, dataflow_id = get_source_dataflow_ids(file_content, item_name) # Look for a parameter that contains the dataflow ID for param in workspace_obj.environment_parameter.get("find_replace", []): # Extract values from the parameter input_type, input_name, input_path = extract_parameter_filters(workspace_obj, param) filter_match = check_replacement( input_type, input_name, input_path, ItemType.DATAFLOW.value, item_name, file_path ) find_info = extract_find_value(param, file_content, filter_match) # Skip if this parameter doesn't match the dataflow ID if find_info["pattern"] != dataflow_id: logger.debug( f"Find value: {find_info['pattern']} does not match the dataflow ID: {dataflow_id}, skipping this parameter" ) continue # Extract the replace value for the current environment replace_value = param.get("replace_value", {}).get(workspace_obj.environment, "") # Pass in get_dataflow_name=True to get the source dataflow name, if it exists source_dataflow_name = extract_replace_value(workspace_obj, replace_value, get_dataflow_name=True) if source_dataflow_name: # Return the source dataflow name along with the IDs return source_dataflow_name, dataflow_workspace_id, dataflow_id return "", "", "" def func_process_file(workspace_obj: FabricWorkspace, item_obj: Item, file_obj: File) -> str: """ Custom file processing for dataflow items. Args: workspace_obj: The FabricWorkspace object. item_obj: The item object. file_obj: The file object. """ # Replace the dataflow ID with the logical ID of the source dataflow in the file content return replace_source_dataflow_ids(workspace_obj, item_obj, file_obj) def replace_source_dataflow_ids(workspace_obj: FabricWorkspace, item_obj: Item, file_obj: File) -> str: """ Replaces both the dataflow ID and workspace ID of the source dataflow with logical values for cross-environment compatibility. Args: workspace_obj: The FabricWorkspace object. item_obj: The item object. file_obj: The file object. """ if str(file_obj.file_path).endswith(".pq"): # Get source dataflow info from the dependency dictionary source_dataflow_info = workspace_obj.dataflow_dependencies.get(item_obj.name, {}) # If the info was tracked, proceed to replace the IDs if source_dataflow_info: source_dataflow_name = source_dataflow_info["source_name"] source_dataflow_workspace_id = source_dataflow_info["source_workspace_id"] source_dataflow_id = source_dataflow_info["source_id"] # Get the logical ID of the source dataflow from repository items logical_id = ( workspace_obj.repository_items.get(ItemType.DATAFLOW.value, {}).get(source_dataflow_name, {}).logical_id ) # Replace the dataflow ID with its logical ID and the workspace ID with the default workspace ID if logical_id: file_obj.contents = file_obj.contents.replace(source_dataflow_id, logical_id) file_obj.contents = file_obj.contents.replace(source_dataflow_workspace_id, constants.DEFAULT_GUID) logger.debug( f"Replaced dataflow ID '{source_dataflow_id}' with logical ID '{logical_id}' and workspace ID " f"'{source_dataflow_workspace_id}' with default workspace ID '{constants.DEFAULT_GUID}' " f"in '{item_obj.name}' file" ) return file_obj.contents def _get_dataflow_publish_order(publisher: "DataflowPublisher") -> list[str]: """Get the ordered list of dataflow names based on dependencies.""" return set_dataflow_publish_order(publisher.fabric_workspace_obj, publisher.item_type) class DataflowPublisher(ItemPublisher): """Publisher for Dataflow items.""" item_type = ItemType.DATAFLOW.value parallel_config = ParallelConfig(enabled=False, ordered_items_func=_get_dataflow_publish_order) """Dataflows must be published in dependency order (sequential)""" def publish_one(self, item_name: str, _item: Item) -> None: """Publish a single Dataflow item.""" self.fabric_workspace_obj._publish_item( item_name=item_name, item_type=self.item_type, func_process_file=func_process_file ) ================================================ FILE: src/fabric_cicd/_items/_datapipeline.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions to process and deploy DataPipeline item.""" import logging import re import dpath from fabric_cicd import FabricWorkspace, constants from fabric_cicd._common._item import Item from fabric_cicd._items._base_publisher import ItemPublisher, ParallelConfig from fabric_cicd._items._manage_dependencies import set_publish_order, set_unpublish_order from fabric_cicd.constants import ItemType logger = logging.getLogger(__name__) def find_referenced_datapipelines(fabric_workspace_obj: FabricWorkspace, file_content: dict, lookup_type: str) -> list: """ Scan through pipeline file json dictionary and find pipeline references (including nested pipelines). Args: fabric_workspace_obj: The FabricWorkspace object. file_content: Dict representation of the pipeline-content file. lookup_type: Finding references in deployed file or repo file (Deployed or Repository). """ item_type = ItemType.DATA_PIPELINE.value reference_list = [] guid_pattern = re.compile(constants.VALID_GUID_REGEX) # Use the dpath library to search through the dictionary for all values that match the GUID pattern for _, value in dpath.search(file_content, "**", yielded=True): if isinstance(value, str): match = guid_pattern.search(value) if match: # If a valid GUID is found, convert it to name. If name is not None, it's a pipeline and will be added to the reference list referenced_id = match.group(0) referenced_name = fabric_workspace_obj._convert_id_to_name( item_type=item_type, generic_id=referenced_id, lookup_type=lookup_type ) # Add pipeline to the reference list if it's not already present if referenced_name and referenced_name not in reference_list: reference_list.append(referenced_name) return reference_list def _get_datapipeline_publish_order(publisher: "DataPipelinePublisher") -> list[str]: """Get the ordered list of data pipeline names based on dependencies.""" return set_publish_order(publisher.fabric_workspace_obj, publisher.item_type, find_referenced_datapipelines) class DataPipelinePublisher(ItemPublisher): """Publisher for Data Pipeline items.""" item_type = ItemType.DATA_PIPELINE.value has_dependency_tracking = True parallel_config = ParallelConfig(enabled=False, ordered_items_func=_get_datapipeline_publish_order) """Pipelines must be published in dependency order (sequential)""" def get_unpublish_order(self, items_to_unpublish: list[str]) -> list[str]: """ Get the ordered list of item names based on dependencies for unpublishing. Args: items_to_unpublish: List of item names to be unpublished. Returns: List of item names in the order they should be unpublished (reverse dependency order). """ return set_unpublish_order( self.fabric_workspace_obj, self.item_type, items_to_unpublish, find_referenced_datapipelines ) def publish_one(self, item_name: str, _item: Item) -> None: """Publish a single Data Pipeline item.""" self.fabric_workspace_obj._publish_item(item_name=item_name, item_type=self.item_type) def pre_publish_all(self) -> None: """Refresh deployed items before publishing to resolve references.""" self.fabric_workspace_obj._refresh_deployed_items() ================================================ FILE: src/fabric_cicd/_items/_environment.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions to process and deploy Environment item.""" import logging import re import dpath import yaml from fabric_cicd import FabricWorkspace, constants from fabric_cicd._common._exceptions import InputError from fabric_cicd._common._fabric_endpoint import handle_retry from fabric_cicd._common._file import File from fabric_cicd._common._item import Item from fabric_cicd._items._base_publisher import ItemPublisher from fabric_cicd.constants import ItemType logger = logging.getLogger(__name__) def _process_environment_file( fabric_workspace_obj: FabricWorkspace, item: Item, file_obj: File, ) -> str: """ Process an Environment item file before it is included in the item definition payload. For ``Setting/Sparkcompute.yml`` this performs ``instance_pool_id`` replacement using the ``spark_pool`` parameter configuration so that the correct pool reference is embedded directly in the YAML sent to the Fabric Items API. All other files are returned unchanged. Args: fabric_workspace_obj: The FabricWorkspace object. item: The Item object representing the Environment. file_obj: The File object being processed. Returns: The (possibly modified) file contents as a string. """ if not (file_obj.file_path.name == "Sparkcompute.yml" and file_obj.file_path.parent.name == "Setting"): return file_obj.contents contents = file_obj.contents if "instance_pool_id" not in contents: return contents yaml_body = yaml.safe_load(contents) if not isinstance(yaml_body, dict): return contents if "instance_pool_id" in yaml_body: yaml_body = _replace_instance_pool_id(fabric_workspace_obj, yaml_body, item.name) return yaml.dump(yaml_body, default_flow_style=False, sort_keys=False) def _replace_instance_pool_id(fabric_workspace_obj: FabricWorkspace, yaml_body: dict, item_name: str) -> dict: """ Replace ``instance_pool_id`` in parsed Sparkcompute YAML with a resolved pool GUID. This function reads ``spark_pool`` parameter mappings from ``fabric_workspace_obj.environment_parameter`` and finds the entry whose ``instance_pool_id`` matches the current YAML value. If an ``item_name`` is provided in the mapping, it must match the current Environment item name; otherwise, the mapping applies globally. The mapped target pool ``name`` and ``type`` are then resolved against the workspace custom pool list returned by the Fabric API, and the resolved pool ``id`` is written back to ``yaml_body["instance_pool_id"]``. Args: fabric_workspace_obj: Workspace context containing environment, parameters, and endpoint configuration. yaml_body: Parsed contents of ``Setting/Sparkcompute.yml``. item_name: Environment item name used for optional per-item mapping filters. Returns: The YAML dictionary, updated if a matching mapping is found; otherwise unchanged. """ from fabric_cicd._parameter._utils import process_environment_key pool_id = yaml_body["instance_pool_id"] if "spark_pool" in fabric_workspace_obj.environment_parameter: pools = fabric_workspace_obj._get_workspace_pools() parameter_dict = fabric_workspace_obj.environment_parameter["spark_pool"] for key in parameter_dict: instance_pool_id = key["instance_pool_id"] replace_value = process_environment_key(fabric_workspace_obj.environment, key["replace_value"]) input_name = key.get("item_name") if instance_pool_id == pool_id and (input_name == item_name or not input_name): pool_config = replace_value[fabric_workspace_obj.environment] resolved_id = _resolve_pool_id( pools, pool_name=pool_config["name"], pool_type=pool_config["type"], ) yaml_body["instance_pool_id"] = resolved_id break return yaml_body def _resolve_pool_id(pools: list[dict], pool_name: str, pool_type: str) -> str: """ Resolve a workspace custom Spark pool ID by pool ``name`` and ``type``. Args: pools: Pool objects from ``GET /spark/pools`` (expected to include ``name``, ``type``, and ``id`` fields). pool_name: Target pool display name. pool_type: Target pool type (for example, ``"Capacity"`` or ``"Workspace"``). Returns: The matching pool GUID. Raises: InputError: If no pool exists with the specified ``name`` and ``type``. """ for pool in pools: if pool["name"] == pool_name and pool["type"] == pool_type: return pool["id"] msg = ( f"Could not resolve custom Spark pool: name='{pool_name}', type='{pool_type}'. " f"No matching pool found in the target workspace." ) raise InputError(msg, logger) def _check_environment_publish_state(fabric_workspace_obj: FabricWorkspace, initial_check: bool = False) -> None: """ Checks the publish state of environments after deployment. Args: fabric_workspace_obj: The FabricWorkspace object. initial_check: Flag to ignore publish failures on initial check. """ ongoing_publish = True iteration = 1 environments = fabric_workspace_obj.repository_items.get(ItemType.ENVIRONMENT.value, {}) filtered_environments = [ k for k in environments if ( # Check exclude regex ( not fabric_workspace_obj.publish_item_name_exclude_regex or not re.search(fabric_workspace_obj.publish_item_name_exclude_regex, k) ) # Check items_to_include list and ( not fabric_workspace_obj.items_to_include or k + ".Environment" in fabric_workspace_obj.items_to_include ) ) ] logger.info(f"Checking Environment Publish State for {filtered_environments}") while ongoing_publish: ongoing_publish = False completed = [] running = [] failed = [] response_state = fabric_workspace_obj.endpoint.invoke( method="GET", url=f"{fabric_workspace_obj.base_api_url}/environments/" ) for item in response_state["body"]["value"]: item_name = item["displayName"] item_state = dpath.get(item, "properties/publishDetails/state", default="").lower() if item_name in filtered_environments: if item_state == "running": running.append(item_name) ongoing_publish = True elif item_state == "success": completed.append(item_name) elif item_state in ["failed", "cancelled"]: failed.append(item_name) if not initial_check: msg = f"Publish {item_state} for Environment '{item_name}'" raise Exception(msg) logger.debug( f"Environment publish states - Running: {running}, Succeeded: {completed}, Failed/Cancelled: {failed}" ) if ongoing_publish: handle_retry( attempt=iteration, base_delay=5, response_retry_after=120, prepend_message=f"{constants.INDENT}Operation in progress.", ) iteration += 1 if not initial_check: logger.info(f"{constants.INDENT}Published: {completed}") def _submit_environment_publish(fabric_workspace_obj: FabricWorkspace, item_name: str) -> None: """ Submit a publish request for an Environment item. Triggers the asynchronous publish of the environment's staged settings and libraries. The publish state is monitored separately by the async publish check hooks. Args: fabric_workspace_obj: The FabricWorkspace object. item_name: Name of the environment item to publish. """ item_type = ItemType.ENVIRONMENT.value item_guid = fabric_workspace_obj.repository_items[item_type][item_name].guid # Publish updated settings - compute settings and libraries (long-running operation) # https://learn.microsoft.com/en-us/rest/api/fabric/environment/items/publish-environment fabric_workspace_obj.endpoint.invoke( method="POST", url=f"{fabric_workspace_obj.base_api_url}/environments/{item_guid}/staging/publish?beta=False", poll_long_running=False, ) logger.info(f"{constants.INDENT}Publish Submitted for Environment '{item_name}'") class EnvironmentPublisher(ItemPublisher): """Publisher for Environment items.""" item_type = ItemType.ENVIRONMENT.value has_async_publish_check = True def publish_one(self, item_name: str, item: Item) -> None: """Publish a single Environment item.""" self.fabric_workspace_obj._publish_item( item_name=item_name, item_type=self.item_type, func_process_file=_process_environment_file, skip_publish_logging=True, ) if item.skip_publish: return _submit_environment_publish(self.fabric_workspace_obj, item_name) def pre_publish_all(self) -> None: """Check environment publish state before publishing.""" _check_environment_publish_state(self.fabric_workspace_obj, True) def post_publish_all_check(self) -> None: """Check environment publish state after all environments have been published.""" _check_environment_publish_state(self.fabric_workspace_obj, False) ================================================ FILE: src/fabric_cicd/_items/_eventhouse.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions to process and deploy Eventhouse item.""" import logging from fabric_cicd._common._item import Item from fabric_cicd._items._base_publisher import ItemPublisher from fabric_cicd.constants import EXCLUDE_PATH_REGEX_MAPPING, ItemType logger = logging.getLogger(__name__) class EventhousePublisher(ItemPublisher): """Publisher for Eventhouse items.""" item_type = ItemType.EVENTHOUSE.value def publish_one(self, item_name: str, _item: Item) -> None: """Publish a single Eventhouse item.""" self.fabric_workspace_obj._publish_item( item_name=item_name, item_type=self.item_type, exclude_path=EXCLUDE_PATH_REGEX_MAPPING.get(self.item_type) ) ================================================ FILE: src/fabric_cicd/_items/_eventstream.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions to process and deploy Eventstream item.""" from fabric_cicd._items._base_publisher import ItemPublisher from fabric_cicd.constants import ItemType class EventstreamPublisher(ItemPublisher): """Publisher for Eventstream items.""" item_type = ItemType.EVENTSTREAM.value ================================================ FILE: src/fabric_cicd/_items/_graphqlapi.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions to process and deploy API for GraphQL item.""" from fabric_cicd._items._base_publisher import ItemPublisher from fabric_cicd.constants import ItemType class GraphQLApiPublisher(ItemPublisher): """Publisher for GraphQL API items.""" item_type = ItemType.GRAPHQL_API.value ================================================ FILE: src/fabric_cicd/_items/_kqldashboard.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions to process and deploy Real-Time Dashboard item.""" import json import logging from fabric_cicd import FabricWorkspace from fabric_cicd._common._exceptions import ParsingError from fabric_cicd._common._file import File from fabric_cicd._common._item import Item from fabric_cicd._items._base_publisher import ItemPublisher from fabric_cicd.constants import ItemType logger = logging.getLogger(__name__) def func_process_file(workspace_obj: FabricWorkspace, item_obj: Item, file_obj: File) -> str: """ Custom file processing for KQL Dashboard items. Args: workspace_obj: The FabricWorkspace object. item_obj: The item object. file_obj: The file object. """ # For KQL Dashboard, we do not need to process the file content return ( replace_cluster_uri(workspace_obj, file_obj) if item_obj.type == ItemType.KQL_DASHBOARD.value else file_obj.contents ) def replace_cluster_uri(fabric_workspace_obj: FabricWorkspace, file_obj: File) -> str: """ Replaces an empty cluster URI value in a Real-Time Dashboard item with the cluster URI associated with its KQL Database source in the raw file content. Args: fabric_workspace_obj: The FabricWorkspace object. file_obj: The file object. """ # Create a dictionary from the raw file json_content_dict = json.loads(file_obj.contents) data_sources = json_content_dict.get("dataSources") # Get the KQL Database items from the deployed items database_items = fabric_workspace_obj.deployed_items.get(ItemType.KQL_DATABASE.value, {}) for data_source in data_sources: if not data_source: msg = "No data sources found in the KQL Dashboard item." raise ParsingError(msg, logger) if data_source.get("clusterUri") == "": database_item_name = data_source.get("name") database_item = database_items.get(database_item_name) if not database_item: msg = f"Cannot find the KQL Database source with name '{database_item_name}' as it is not yet deployed." raise ParsingError(msg, logger) database_item_guid = database_item.guid # Get the cluster URI of the KQL database kqldatabase_data = fabric_workspace_obj.endpoint.invoke( method="GET", url=f"{fabric_workspace_obj.base_api_url}/kqlDatabases/{database_item_guid}", ) kqldatabase_cluster_uri = kqldatabase_data.get("body", {}).get("properties", {}).get("queryServiceUri") # Replace the cluster URI value if not kqldatabase_cluster_uri: msg = f"Cluster URI for KQL Database '{database_item_name}' is not found." raise ParsingError(msg, logger) data_source["clusterUri"] = kqldatabase_cluster_uri return json.dumps(json_content_dict, indent=2) class KQLDashboardPublisher(ItemPublisher): """Publisher for KQL Dashboard items.""" item_type = ItemType.KQL_DASHBOARD.value def publish_one(self, item_name: str, _item: Item) -> None: """Publish a single KQL Dashboard item.""" self.fabric_workspace_obj._publish_item( item_name=item_name, item_type=self.item_type, func_process_file=func_process_file ) def pre_publish_all(self) -> None: """Refresh deployed items to get KQL Database cluster URIs.""" self.fabric_workspace_obj._refresh_deployed_items() ================================================ FILE: src/fabric_cicd/_items/_kqldatabase.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions to process and deploy KQL Database item.""" from fabric_cicd._items._base_publisher import ItemPublisher from fabric_cicd.constants import ItemType class KQLDatabasePublisher(ItemPublisher): """Publisher for KQL Database items.""" item_type = ItemType.KQL_DATABASE.value ================================================ FILE: src/fabric_cicd/_items/_kqlqueryset.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions to process and deploy KQL Queryset item.""" import json import logging from fabric_cicd import FabricWorkspace from fabric_cicd._common._exceptions import ParsingError from fabric_cicd._common._file import File from fabric_cicd._common._item import Item from fabric_cicd._items._base_publisher import ItemPublisher from fabric_cicd.constants import ItemType logger = logging.getLogger(__name__) def func_process_file(workspace_obj: FabricWorkspace, item_obj: Item, file_obj: File) -> str: """ Custom file processing for kql queryset items. Args: workspace_obj: The FabricWorkspace object. item_obj: The item object. file_obj: The file object. """ return ( replace_cluster_uri(workspace_obj, file_obj) if item_obj.type == ItemType.KQL_QUERYSET.value else file_obj.contents ) def replace_cluster_uri(fabric_workspace_obj: FabricWorkspace, file_obj: File) -> str: """ Replaces an empty cluster URI value in a KQL Queryset item with the cluster URI associated with its KQL Database source in the raw file content. Args: fabric_workspace_obj: The FabricWorkspace object. file_obj: The file object. """ # Create a dictionary from the raw file json_content_dict = json.loads(file_obj.contents) queryset = json_content_dict.get("queryset") data_sources = queryset.get("dataSources") if queryset else None if not data_sources: logger.debug("No data sources found in KQL Queryset.") return file_obj.contents # Get the KQL Database items from the deployed items database_items = fabric_workspace_obj.deployed_items.get(ItemType.KQL_DATABASE.value, {}) # If the cluster URI is empty, replace it with the cluster URI of the KQL database for data_source in data_sources: if data_source.get("clusterUri") == "": database_item_name = data_source.get("databaseItemName") logger.debug(f"Found empty cluster URI for database '{database_item_name}'") database_item = database_items.get(database_item_name) if not database_item: msg = f"Cannot find the KQL Database source with name '{database_item_name}' as it is not yet deployed." raise ParsingError(msg, logger) database_item_guid = database_item.guid # Get the cluster URI of the KQL database kqldatabase_data = fabric_workspace_obj.endpoint.invoke( method="GET", url=f"{fabric_workspace_obj.base_api_url}/kqlDatabases/{database_item_guid}", ) try: kqldatabase_cluster_uri = kqldatabase_data["body"]["properties"]["queryServiceUri"] except (KeyError, TypeError): kqldatabase_cluster_uri = None if not kqldatabase_cluster_uri: msg = f"Cannot find the cluster URI for KQL Database '{database_item_name}'." raise ParsingError(msg, logger) # Replace the cluster URI value data_source["clusterUri"] = kqldatabase_cluster_uri logger.debug( f"Updated the cluster URI for data source '{database_item_name}' with '{kqldatabase_cluster_uri}'" ) logger.debug("Successfully updated all empty cluster URIs.") return json.dumps(json_content_dict, indent=2) class KQLQuerysetPublisher(ItemPublisher): """Publisher for KQL Queryset items.""" item_type = ItemType.KQL_QUERYSET.value def publish_one(self, item_name: str, _item: Item) -> None: """Publish a single KQL Queryset item.""" self.fabric_workspace_obj._publish_item( item_name=item_name, item_type=self.item_type, func_process_file=func_process_file ) def pre_publish_all(self) -> None: """Refresh deployed items to get KQL Database cluster URIs.""" self.fabric_workspace_obj._refresh_deployed_items() ================================================ FILE: src/fabric_cicd/_items/_lakehouse.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions to process and deploy Lakehouse item.""" import json import logging import dpath from fabric_cicd import FabricWorkspace, constants from fabric_cicd._common._exceptions import FailedPublishedItemStatusError from fabric_cicd._common._fabric_endpoint import handle_retry from fabric_cicd._common._item import Item from fabric_cicd._items._base_publisher import ItemPublisher, Publisher from fabric_cicd.constants import FeatureFlag, ItemType logger = logging.getLogger(__name__) def check_sqlendpoint_provision_status(fabric_workspace_obj: FabricWorkspace, item_obj: Item) -> None: """ Check the SQL endpoint status of the published lakehouses Args: fabric_workspace_obj: The FabricWorkspace object containing the items to be published item_obj: The item object to check the SQL endpoint status for """ iteration = 1 while True: sql_endpoint_status = None response_state = fabric_workspace_obj.endpoint.invoke( method="GET", url=f"{fabric_workspace_obj.base_api_url}/lakehouses/{item_obj.guid}" ) sql_endpoint_status = dpath.get( response_state, "body/properties/sqlEndpointProperties/provisioningStatus", default=None ) if sql_endpoint_status == "Success": logger.info(f"{constants.INDENT}SQL Endpoint provisioned successfully") break if sql_endpoint_status == "Failed": msg = f"Cannot resolve SQL endpoint for lakehouse {item_obj.name}" raise FailedPublishedItemStatusError(msg, logger) handle_retry( attempt=iteration, base_delay=5, response_retry_after=30, prepend_message=f"{constants.INDENT}SQL Endpoint provisioning in progress", ) iteration += 1 def list_deployed_shortcuts(fabric_workspace_obj: FabricWorkspace, item_obj: Item) -> list: """ Lists all deployed shortcut paths Args: fabric_workspace_obj: The FabricWorkspace object containing the items to be published item_obj: The item object to list the shortcuts for """ request_url = f"{fabric_workspace_obj.base_api_url}/items/{item_obj.guid}/shortcuts" deployed_shortcut_paths = [] while request_url: # https://learn.microsoft.com/en-us/rest/api/fabric/core/onelake-shortcuts/list-shortcuts response = fabric_workspace_obj.endpoint.invoke(method="GET", url=request_url) # Handle cases where the response body is empty shortcuts = response["body"].get("value", []) deployed_shortcut_paths.extend(f"{shortcut['path']}/{shortcut['name']}" for shortcut in shortcuts) request_url = response["header"].get("continuationUri", None) return deployed_shortcut_paths def replace_default_lakehouse_id(shortcut: dict, item_obj: Item) -> dict: """ Replaces the default lakehouse ID (all zeros) with the actual lakehouse ID in the shortcut definition when present. Args: shortcut: The shortcut definition dictionary item_obj: The item object used to get the default lakehouse ID """ if dpath.get(shortcut, "target/oneLake/itemId", default=None) == constants.DEFAULT_GUID: shortcut["target"]["oneLake"]["itemId"] = item_obj.guid return shortcut class LakehousePublisher(ItemPublisher): """Publisher for Lakehouse items.""" item_type = ItemType.LAKEHOUSE.value def publish_one(self, item_name: str, item: Item) -> None: """Publish a single Lakehouse item.""" creation_payload = next( ( {"enableSchemas": True} for file in item.item_files if file.name == "lakehouse.metadata.json" and "defaultSchema" in file.contents ), None, ) self.fabric_workspace_obj._publish_item( item_name=item_name, item_type=self.item_type, creation_payload=creation_payload, skip_publish_logging=True, ) # Check if the item is published to avoid any post publish actions if item.skip_publish: return check_sqlendpoint_provision_status(self.fabric_workspace_obj, item) logger.info(f"{constants.INDENT}Published Lakehouse '{item_name}'") def post_publish_all(self) -> None: """Publish shortcuts after all lakehouses are published to protect interrelationships.""" if FeatureFlag.ENABLE_SHORTCUT_PUBLISH.value in constants.FEATURE_FLAG: for item_obj in self.fabric_workspace_obj.repository_items.get(self.item_type, {}).values(): # Check if the item is published to avoid any post publish actions if not item_obj.skip_publish and item_obj.guid: shortcut_publisher = ShortcutPublisher(self.fabric_workspace_obj, item_obj) shortcut_publisher.publish_all() class ShortcutPublisher(Publisher): """Publisher for Lakehouse shortcuts.""" def __init__(self, fabric_workspace_obj: FabricWorkspace, item_obj: Item) -> None: """ Initialize the shortcut publisher. Args: fabric_workspace_obj: The FabricWorkspace object containing the items to be published. item_obj: The lakehouse item object to publish shortcuts for. """ super().__init__(fabric_workspace_obj) self.item_obj = item_obj def _unpublish_shortcuts(self, shortcut_paths: list) -> None: """ Unpublish shortcuts from the lakehouse. Args: shortcut_paths: The list of shortcut paths to unpublish. """ for deployed_shortcut_path in shortcut_paths: # https://learn.microsoft.com/en-us/rest/api/fabric/core/onelake-shortcuts/delete-shortcut self.fabric_workspace_obj.endpoint.invoke( method="DELETE", url=f"{self.fabric_workspace_obj.base_api_url}/items/{self.item_obj.guid}/shortcuts/{deployed_shortcut_path}", ) def publish_one(self, _shortcut_name: str, shortcut: dict) -> None: """ Publish a single shortcut. Args: _shortcut_name: The name/path of the shortcut to publish. shortcut: The shortcut definition to publish. """ shortcut = replace_default_lakehouse_id(shortcut, self.item_obj) # https://learn.microsoft.com/en-us/rest/api/fabric/core/onelake-shortcuts/create-shortcut try: self.fabric_workspace_obj.endpoint.invoke( method="POST", url=f"{self.fabric_workspace_obj.base_api_url}/items/{self.item_obj.guid}/shortcuts?shortcutConflictPolicy=CreateOrOverwrite", body=shortcut, ) logger.info(f"{constants.INDENT}Published Shortcut '{shortcut['name']}'") except Exception as e: if FeatureFlag.CONTINUE_ON_SHORTCUT_FAILURE.value in constants.FEATURE_FLAG: logger.warning( f"Failed to publish Shortcut '{shortcut['name']}'. This usually happens when the lakehouse containing the source for this shortcut is published as a shell and has no data yet." ) logger.info("The publish process will continue with the other items.") return msg = f"Failed to publish '{shortcut['name']}' for lakehouse {self.item_obj.name}" raise FailedPublishedItemStatusError(msg, logger) from e def publish_all(self) -> None: """ Publish all shortcuts for the lakehouse item. Loads shortcuts from metadata, filters based on exclude regex, unpublishes orphaned shortcuts, and publishes all remaining shortcuts. """ from fabric_cicd._common._check_utils import check_regex deployed_shortcuts = list_deployed_shortcuts(self.fabric_workspace_obj, self.item_obj) shortcut_file_obj = next( (file for file in self.item_obj.item_files if file.name == "shortcuts.metadata.json"), None ) if shortcut_file_obj: shortcut_file_obj.contents = self.fabric_workspace_obj._replace_parameters(shortcut_file_obj, self.item_obj) shortcut_file_obj.contents = self.fabric_workspace_obj._replace_logical_ids(shortcut_file_obj.contents) shortcut_file_obj.contents = self.fabric_workspace_obj._replace_workspace_ids(shortcut_file_obj.contents) shortcuts = json.loads(shortcut_file_obj.contents) or [] else: logger.debug("No shortcuts.metadata.json found") shortcuts = [] # Filter shortcuts based on exclude regex if provided if self.fabric_workspace_obj.shortcut_exclude_regex: regex_pattern = check_regex(self.fabric_workspace_obj.shortcut_exclude_regex) original_count = len(shortcuts) excluded_shortcuts = [s["name"] for s in shortcuts if "name" in s and regex_pattern.match(s["name"])] shortcuts = [s for s in shortcuts if "name" in s and not regex_pattern.match(s["name"])] excluded_count = original_count - len(shortcuts) if excluded_count > 0: logger.info( f"{constants.INDENT}Excluded {excluded_count} shortcut(s) from {self.item_obj.name} deployment based on regex pattern" ) logger.info(f"{constants.INDENT}Excluded shortcuts: {excluded_shortcuts}") shortcuts_to_publish = {f"{shortcut['path']}/{shortcut['name']}": shortcut for shortcut in shortcuts} if shortcuts_to_publish: logger.info(f"Publishing Lakehouse '{self.item_obj.name}' Shortcuts") shortcut_paths_to_unpublish = [path for path in deployed_shortcuts if path not in shortcuts_to_publish] self._unpublish_shortcuts(shortcut_paths_to_unpublish) # Deploy and overwrite shortcuts for shortcut_path, shortcut in shortcuts_to_publish.items(): self.publish_one(shortcut_path, shortcut) ================================================ FILE: src/fabric_cicd/_items/_manage_dependencies.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions to process items with dependencies.""" import base64 import json import logging from collections import defaultdict, deque from pathlib import Path from typing import Callable from fabric_cicd import FabricWorkspace, constants from fabric_cicd._common._exceptions import ParsingError logger = logging.getLogger(__name__) def set_publish_order( fabric_workspace_obj: FabricWorkspace, item_type: str, find_referenced_items_func: Callable ) -> list: """ Creates a publish order list for items of the same type, considering their dependencies. Args: fabric_workspace_obj: The FabricWorkspace object. item_type: Type of item to order (e.g., 'DataPipeline'). find_referenced_items_func: Function to find referenced items in content. """ # Get all items of the given type from the repository items = fabric_workspace_obj.repository_items.get(item_type, {}) # Construct the unsorted_dict with an item and its associated file content unsorted_dict = {} # Set the file name based on the item type (e.g., 'pipeline-content.json' for DataPipeline) file_name = constants.ITEM_TYPE_TO_FILE[item_type] for item_name, item_details in items.items(): with Path(item_details.path, file_name).open(encoding="utf-8") as f: raw_file = f.read() # If the file is a JSON, load as dict; otherwise, keep as the raw file item_content = json.loads(raw_file) if file_name.endswith(".json") else raw_file unsorted_dict[item_name] = item_content # Return a list of items sorted by their dependencies return sort_items(fabric_workspace_obj, unsorted_dict, "Repository", find_referenced_items_func) def set_unpublish_order( fabric_workspace_obj: FabricWorkspace, item_type: str, unpublish_list: list, find_referenced_items_func: Callable, ) -> list: """ Creates an unpublish order list for items of the same type, considering their dependencies. Args: fabric_workspace_obj: The FabricWorkspace object. item_type: Type of item to order (e.g., 'DataPipeline'). unpublish_list: List of items to unpublish. find_referenced_items_func: Function to find referenced items in content. """ unsorted_item_dict = {} file_name = constants.ITEM_TYPE_TO_FILE[item_type] for item_name in unpublish_list: # Get deployed item definition # https://learn.microsoft.com/en-us/rest/api/fabric/core/items/get-item-definition item_guid = fabric_workspace_obj.deployed_items[item_type][item_name].guid response = fabric_workspace_obj.endpoint.invoke( method="POST", url=f"{fabric_workspace_obj.base_api_url}/items/{item_guid}/getDefinition" ) for part in response["body"]["definition"]["parts"]: if part["path"] == file_name: # Decode Base64 string to dictionary decoded_bytes = base64.b64decode(part["payload"]) decoded_string = decoded_bytes.decode("utf-8") unsorted_item_dict[item_name] = ( json.loads(decoded_string) if file_name.endswith(".json") else decoded_string ) break # Determine order to delete w/o dependencies return sort_items(fabric_workspace_obj, unsorted_item_dict, "Deployed", find_referenced_items_func) def sort_items( fabric_workspace_obj: FabricWorkspace, unsorted_dict: dict, lookup_type: str, find_referenced_items_func: Callable ) -> list: """ Performs topological sort on items of a given item type based on their dependencies. Args: fabric_workspace_obj: The FabricWorkspace object. unsorted_dict: Dictionary mapping items to their file content. lookup_type: Finding references in deployed file or repo file (Deployed or Repository). find_referenced_items_func: Function to find referenced items in content. """ # Step 1: Create a graph to manage dependencies graph = defaultdict(list) in_degree = defaultdict(int) unpublish_items = [] # Step 2: Build the graph and count the in-degrees for item_name, item_content in unsorted_dict.items(): logger.debug(f"Processing item: '{item_name}'") # In an unpublish case, keep track of items to get unpublished if lookup_type == "Deployed": unpublish_items.append(item_name) referenced_items = find_referenced_items_func(fabric_workspace_obj, item_content, lookup_type) for referenced_name in referenced_items: graph[referenced_name].append(item_name) in_degree[item_name] += 1 # Ensure every item has an entry in the in-degree map if item_name not in in_degree: in_degree[item_name] = 0 logger.debug(f"Graph: {graph}") logger.debug(f"In-degree map: {in_degree}") # In an unpublish case, adjust in_degree to include entire dependency chain if lookup_type == "Deployed": for item_name in graph: if item_name not in in_degree: in_degree[item_name] = 0 for neighbor in graph[item_name]: if neighbor not in in_degree: in_degree[neighbor] += 1 # Step 3: Perform a topological sort to determine the correct publish order zero_in_degree_queue = deque([item_name for item_name in in_degree if in_degree[item_name] == 0]) sorted_items = [] logger.debug(f"Zero_in_degree_queue: {zero_in_degree_queue}") while zero_in_degree_queue: item_name = zero_in_degree_queue.popleft() sorted_items.append(item_name) for neighbor in graph[item_name]: in_degree[neighbor] -= 1 if in_degree[neighbor] == 0: zero_in_degree_queue.append(neighbor) if len(sorted_items) != len(in_degree): msg = "There is a cycle in the graph. Cannot determine a valid publish order." raise ParsingError(msg, logger) # Remove items not present in unpublish list and invert order for deployed sort if lookup_type == "Deployed": sorted_items = [item_name for item_name in sorted_items if item_name in unpublish_items] sorted_items = sorted_items[::-1] logger.debug(f"Sorted items in {lookup_type}: {sorted_items}") return sorted_items ================================================ FILE: src/fabric_cicd/_items/_mirroreddatabase.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions to process and deploy Mirrored Database item.""" from fabric_cicd._items._base_publisher import ItemPublisher from fabric_cicd.constants import ItemType class MirroredDatabasePublisher(ItemPublisher): """Publisher for Mirrored Database items.""" item_type = ItemType.MIRRORED_DATABASE.value ================================================ FILE: src/fabric_cicd/_items/_mlexperiment.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions to process and deploy ML Experiment item.""" from fabric_cicd._items._base_publisher import ItemPublisher from fabric_cicd.constants import ItemType class MLExperimentPublisher(ItemPublisher): """Publisher for ML Experiment items.""" item_type = ItemType.ML_EXPERIMENT.value ================================================ FILE: src/fabric_cicd/_items/_mounteddatafactory.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions to process and deploy Mounted Data Factory item.""" from fabric_cicd._items._base_publisher import ItemPublisher from fabric_cicd.constants import ItemType class MountedDataFactoryPublisher(ItemPublisher): """Publisher for Mounted Data Factory items.""" item_type = ItemType.MOUNTED_DATA_FACTORY.value ================================================ FILE: src/fabric_cicd/_items/_notebook.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions to process and deploy Notebook item.""" from fabric_cicd._common._item import Item from fabric_cicd._items._base_publisher import ItemPublisher from fabric_cicd.constants import API_FORMAT_MAPPING, ItemType class NotebookPublisher(ItemPublisher): """Publisher for Notebook items.""" item_type = ItemType.NOTEBOOK.value def publish_one(self, item_name: str, item: Item) -> None: """Publish a Notebook item.""" is_ipynb = any(file.file_path.suffix == ".ipynb" for file in item.item_files) # Sort files to ensure consistent payload order for Fabric API notebook processing # Fabric API expects content file (.py) to be processed before settings file (.json) when both are present def _sort_key(f: Item) -> tuple[int, str]: # .ipynb included for completeness; in practice, .json settings only exist with .py notebooks (git integrated format) priority = {".platform": 0, ".py": 1, ".ipynb": 1, ".json": 3} # Account for other file types that may be added later to the notebook item and assign to priority 2 return (priority.get(f.file_path.name, priority.get(f.file_path.suffix, 2)), f.file_path.name) item.item_files.sort(key=_sort_key) kwargs = {} if is_ipynb: api_format = API_FORMAT_MAPPING.get(self.item_type) if api_format: kwargs["api_format"] = api_format self.fabric_workspace_obj._publish_item( item_name=item_name, item_type=self.item_type, **kwargs, ) ================================================ FILE: src/fabric_cicd/_items/_ontology.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions to process and deploy Ontology item.""" from fabric_cicd._items._base_publisher import ItemPublisher from fabric_cicd.constants import ItemType class OntologyPublisher(ItemPublisher): """Publisher for Ontology items.""" item_type = ItemType.ONTOLOGY.value ================================================ FILE: src/fabric_cicd/_items/_report.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions to process and deploy Report item.""" import json import logging from fabric_cicd import FabricWorkspace from fabric_cicd._common._exceptions import ItemDependencyError from fabric_cicd._common._file import File from fabric_cicd._common._item import Item from fabric_cicd._items._base_publisher import ItemPublisher from fabric_cicd.constants import EXCLUDE_PATH_REGEX_MAPPING, ItemType logger = logging.getLogger(__name__) def func_process_file(workspace_obj: FabricWorkspace, item_obj: Item, file_obj: File) -> str: """ Custom file processing for report items. Args: workspace_obj: The FabricWorkspace object. item_obj: The item object. file_obj: The file object. """ if file_obj.name == "definition.pbir": definition_body = json.loads(file_obj.contents) if ( "datasetReference" in definition_body and "byPath" in definition_body["datasetReference"] and definition_body["datasetReference"]["byPath"] is not None ): model_rel_path = definition_body["datasetReference"]["byPath"]["path"] model_path = str((item_obj.path / model_rel_path).resolve()) model_id = workspace_obj._convert_path_to_id(ItemType.SEMANTIC_MODEL.value, model_path) if not model_id: msg = "Semantic model not found in the repository. Cannot deploy a report with a relative path without deploying the model." raise ItemDependencyError(msg, logger) definition_body["$schema"] = ( "https://developer.microsoft.com/json-schemas/fabric/item/report/definitionProperties/1.0.0/schema.json" ) definition_body["datasetReference"] = { "byConnection": { "connectionString": None, "pbiServiceModelId": None, "pbiModelVirtualServerName": "sobe_wowvirtualserver", "pbiModelDatabaseName": f"{model_id}", "name": "EntityDataSource", "connectionType": "pbiServiceXmlaStyleLive", } } return json.dumps(definition_body, indent=4) return file_obj.contents class ReportPublisher(ItemPublisher): """Publisher for Report items.""" item_type = ItemType.REPORT.value def publish_one(self, item_name: str, _item: Item) -> None: """Publish a single Report item.""" self.fabric_workspace_obj._publish_item( item_name=item_name, item_type=self.item_type, exclude_path=EXCLUDE_PATH_REGEX_MAPPING.get(self.item_type), func_process_file=func_process_file, ) ================================================ FILE: src/fabric_cicd/_items/_semanticmodel.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions to process and deploy Semantic Model item.""" import logging from fabric_cicd import FabricWorkspace, constants from fabric_cicd._common._item import Item from fabric_cicd._items._base_publisher import ItemPublisher from fabric_cicd._parameter._utils import process_environment_key from fabric_cicd.constants import EXCLUDE_PATH_REGEX_MAPPING, ItemType logger = logging.getLogger(__name__) def build_binding_mapping_legacy(fabric_workspace_obj: FabricWorkspace, semantic_model_binding: list) -> dict: """ Build the connection mapping from legacy list-based semantic_model_binding parameter. Args: fabric_workspace_obj: The FabricWorkspace object semantic_model_binding: The semantic_model_binding parameter as a list Returns: Dictionary mapping semantic model names to connection IDs """ logger.warning( "The legacy 'semantic_model_binding' list format is deprecated and will be removed in a future release. " "Please migrate to the new dictionary format with 'default' and 'models' keys. " "See: https://microsoft.github.io/fabric-cicd/how_to/parameterization/" ) item_type = "SemanticModel" binding_mapping = {} repository_models = set(fabric_workspace_obj.repository_items.get(item_type, {}).keys()) for entry in semantic_model_binding: connection_id = entry.get("connection_id") model_names = entry.get("semantic_model_name", []) if not connection_id: logger.debug("No connection_id found in semantic_model_binding entry, skipping") continue # Legacy format only supports string connection_id if isinstance(connection_id, dict): logger.warning( "Environment-specific connection_id dictionaries are not supported in the legacy format. " "Please migrate to the new dictionary format to use environment-specific values." ) continue if isinstance(model_names, str): model_names = [model_names] for name in model_names: if name not in repository_models: logger.warning(f"Semantic model '{name}' specified in parameter.yml not found in repository") continue binding_mapping[name] = connection_id return binding_mapping def build_binding_mapping( fabric_workspace_obj: FabricWorkspace, semantic_model_binding: dict, environment: str ) -> dict: """ Build the connection mapping from semantic_model_binding parameter. The new format requires environment-specific connection_id values (use '_ALL_' for all environments). Supports: - default.connection_id: Applied to all models in the repository that are not explicitly listed - models: List of explicit model-to-connection mappings Args: fabric_workspace_obj: The FabricWorkspace object semantic_model_binding: The semantic_model_binding parameter dictionary environment: The target environment name (_ALL_ key can be used) Returns: Dictionary mapping semantic model names to connection IDs """ item_type = "SemanticModel" binding_mapping = {} repository_models = set(fabric_workspace_obj.repository_items.get(item_type, {}).keys()) # Get default connection_id for this environment default_connection_id = None default_config = semantic_model_binding.get("default", {}) if default_config: connection_id_config = default_config.get("connection_id", {}) connection_id_config = process_environment_key(environment, connection_id_config) default_connection_id = connection_id_config.get(environment) if not default_connection_id: logger.debug(f"Environment '{environment}' not found in default.connection_id") # Process explicit model bindings explicit_models = set() models_config = semantic_model_binding.get("models", []) for model in models_config: model_names = model.get("semantic_model_name", []) connection_id_config = model.get("connection_id", {}) if isinstance(model_names, str): model_names = [model_names] connection_id_config = process_environment_key(environment, connection_id_config) connection_id = connection_id_config.get(environment) if not connection_id: logger.debug(f"Environment '{environment}' not found in connection_id for semantic model(s): {model_names}") continue # Track models with explicit bindings to exclude from default connection assignment explicit_models.update(model_names) for name in model_names: if name not in repository_models: logger.warning(f"Semantic model '{name}' specified in parameter.yml not found in repository") continue binding_mapping[name] = connection_id # Apply default connection to non-explicit models if default_connection_id: default_models = repository_models - explicit_models for model_name in default_models: binding_mapping[model_name] = default_connection_id logger.debug(f"Applying default connection to semantic model '{model_name}'") return binding_mapping def get_connections(fabric_workspace_obj: FabricWorkspace) -> dict: """ Get all connections from the workspace. Args: fabric_workspace_obj: The FabricWorkspace object Returns: Dictionary with connection ID as key and connection details as value """ # https://learn.microsoft.com/en-us/rest/api/fabric/core/connections/list-connections connections_url = f"{constants.FABRIC_API_ROOT_URL}/v1/connections" try: response = fabric_workspace_obj.endpoint.invoke(method="GET", url=connections_url) connections_list = response.get("body", {}).get("value", []) connections_dict = {} for connection in connections_list: connection_id = connection.get("id") if connection_id: connections_dict[connection_id] = { "id": connection_id, "connectivityType": connection.get("connectivityType"), "connectionDetails": connection.get("connectionDetails", {}), } return connections_dict except Exception as e: logger.error(f"Failed to retrieve connections: {e}") return {} def bind_semanticmodel_to_connection( fabric_workspace_obj: FabricWorkspace, connections: dict, connection_details: dict ) -> None: """ Binds semantic models to their specified connections. Args: fabric_workspace_obj: The FabricWorkspace object containing the items to be published. connections: Dictionary of connection objects with connection ID as key. connection_details: Dictionary mapping semantic model names to connection IDs from parameter.yml. """ item_type = ItemType.SEMANTIC_MODEL.value for model_name, connection_id in connection_details.items(): # Check if the connection ID exists in the connections dict if connection_id not in connections: logger.warning(f"Connection ID '{connection_id}' not found for semantic model '{model_name}'") continue # Get the semantic model object (validated during binding mapping creation) item_obj = fabric_workspace_obj.repository_items[item_type][model_name] # Skip models excluded by items_to_include (skip_publish=True) or with no deployed # GUID — binding would produce an empty-ID URL (HTTP 400) and fail with a server error. if item_obj.skip_publish or not item_obj.guid: logger.debug( f"Skipping connection binding for semantic model '{model_name}' " f"(skip_publish={item_obj.skip_publish}, guid='{item_obj.guid}')" ) continue model_id = item_obj.guid logger.info(f"Binding semantic model '{model_name}' (ID: {model_id}) to connection '{connection_id}'") try: # Get the connection details for this semantic model from Fabric API # https://learn.microsoft.com/en-us/rest/api/fabric/core/items/list-item-connections item_connections_url = f"{constants.FABRIC_API_ROOT_URL}/v1/workspaces/{fabric_workspace_obj.workspace_id}/items/{model_id}/connections" connections_response = fabric_workspace_obj.endpoint.invoke(method="GET", url=item_connections_url) connections_data = connections_response.get("body", {}).get("value", []) if not connections_data: logger.debug(f"No existing connections found for semantic model '{model_name}', skipping binding") continue # Use the first connection as the template connection_binding = connections_data[0] # Update the connection binding with the target connection ID from parameter.yml connection_binding["id"] = connection_id connection_binding["connectivityType"] = connections[connection_id]["connectivityType"] connection_binding["connectionDetails"] = connections[connection_id]["connectionDetails"] # Build the request body request_body = build_request_body({"connectionBinding": connection_binding}) # Make the bind connection API call # https://learn.microsoft.com/en-us/rest/api/fabric/semanticmodel/items/bind-semantic-model-connection binding_url = f"{constants.FABRIC_API_ROOT_URL}/v1/workspaces/{fabric_workspace_obj.workspace_id}/semanticModels/{model_id}/bindConnection" bind_response = fabric_workspace_obj.endpoint.invoke( method="POST", url=binding_url, body=request_body, ) status_code = bind_response.get("status_code") if status_code == 200: logger.info(f"Successfully bound semantic model '{model_name}' to connection '{connection_id}'") else: logger.warning(f"Failed to bind semantic model '{model_name}'. Status code: {status_code}") except Exception as e: logger.error(f"Failed to bind semantic model '{model_name}' to connection: {e!s}") continue def build_request_body(body: dict) -> dict: """ Build request body with specific order of fields for connection binding. Args: body: Dictionary containing connectionBinding data Returns: Ordered dictionary with id, connectivityType, and connectionDetails """ connection_binding = body.get("connectionBinding", {}) connection_details = connection_binding.get("connectionDetails", {}) return { "connectionBinding": { "id": connection_binding.get("id"), "connectivityType": connection_binding.get("connectivityType"), "connectionDetails": { "type": connection_details.get("type") if "type" in connection_details else None, "path": connection_details.get("path") if "path" in connection_details else None, }, } } class SemanticModelPublisher(ItemPublisher): """Publisher for Semantic Model items.""" item_type = ItemType.SEMANTIC_MODEL.value def publish_one(self, item_name: str, _item: Item) -> None: """Publish a single Semantic Model item.""" self.fabric_workspace_obj._publish_item( item_name=item_name, item_type=self.item_type, exclude_path=EXCLUDE_PATH_REGEX_MAPPING.get(self.item_type) ) def post_publish_all(self) -> None: """Bind semantic models to connections after all models are published.""" semantic_model_binding = self.fabric_workspace_obj.environment_parameter.get("semantic_model_binding", {}) if not semantic_model_binding: return # Build connection mapping from semantic_model_binding parameter (support legacy or new formats) environment = self.fabric_workspace_obj.environment if isinstance(semantic_model_binding, list): binding_mapping = build_binding_mapping_legacy(self.fabric_workspace_obj, semantic_model_binding) elif isinstance(semantic_model_binding, dict): binding_mapping = build_binding_mapping(self.fabric_workspace_obj, semantic_model_binding, environment) else: logger.warning( f"Invalid 'semantic_model_binding' type: {type(semantic_model_binding).__name__}. " "Expected list or dict. Skipping semantic model binding." ) return if binding_mapping: connections = get_connections(self.fabric_workspace_obj) bind_semanticmodel_to_connection( fabric_workspace_obj=self.fabric_workspace_obj, connections=connections, connection_details=binding_mapping, ) ================================================ FILE: src/fabric_cicd/_items/_sparkjobdefinition.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions to process and deploy Spark Job Definition item.""" import logging from fabric_cicd._common._item import Item from fabric_cicd._items._base_publisher import ItemPublisher from fabric_cicd.constants import API_FORMAT_MAPPING, ItemType logger = logging.getLogger(__name__) class SparkJobDefinitionPublisher(ItemPublisher): """Publisher for Spark Job Definition items.""" item_type = ItemType.SPARK_JOB_DEFINITION.value def publish_one(self, item_name: str, _item: Item) -> None: """Publish a single Spark Job Definition item.""" self.fabric_workspace_obj._publish_item( item_name=item_name, item_type=self.item_type, api_format=API_FORMAT_MAPPING.get(self.item_type) ) ================================================ FILE: src/fabric_cicd/_items/_sqldatabase.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions to process and deploy SQL Database item.""" import logging from fabric_cicd import constants from fabric_cicd._common._item import Item from fabric_cicd._items._base_publisher import ItemPublisher from fabric_cicd.constants import ItemType logger = logging.getLogger(__name__) class SQLDatabasePublisher(ItemPublisher): """Publisher for SQL Database items.""" item_type = ItemType.SQL_DATABASE.value def publish_one(self, item_name: str, item: Item) -> None: """Publish a single SQL Database item.""" self.fabric_workspace_obj._publish_item( item_name=item_name, item_type=self.item_type, skip_publish_logging=True, ) # Check if the item is published to avoid any post publish actions if item.skip_publish: return logger.info(f"{constants.INDENT}Published SQLDatabase '{item_name}'") ================================================ FILE: src/fabric_cicd/_items/_userdatafunction.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions to process and deploy User Data Function item.""" from fabric_cicd._items._base_publisher import ItemPublisher from fabric_cicd.constants import ItemType class UserDataFunctionPublisher(ItemPublisher): """Publisher for User Data Function items.""" item_type = ItemType.USER_DATA_FUNCTION.value ================================================ FILE: src/fabric_cicd/_items/_variablelibrary.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions to process and deploy Variable Library item.""" import json import logging from fabric_cicd import FabricWorkspace, constants from fabric_cicd._common._item import Item from fabric_cicd._items._base_publisher import ItemPublisher from fabric_cicd.constants import ItemType logger = logging.getLogger(__name__) def activate_value_set(fabric_workspace_obj: FabricWorkspace, item_obj: Item) -> None: """ Activates the value set for the given Variable Library item. Args: fabric_workspace_obj: The FabricWorkspace object. item_obj: The item object. """ settings_file_obj = next((file for file in item_obj.item_files if file.name == "settings.json"), None) if settings_file_obj: settings_dict = json.loads(settings_file_obj.contents) if fabric_workspace_obj.environment in settings_dict["valueSetsOrder"]: active_value_set = fabric_workspace_obj.environment else: active_value_set = "Default value set" logger.warning( f"Provided target environment '{fabric_workspace_obj.environment}' does not match any value sets. Using '{active_value_set}'" ) body = {"properties": {"activeValueSetName": active_value_set}} fabric_workspace_obj.endpoint.invoke( method="PATCH", url=f"{fabric_workspace_obj.base_api_url}/VariableLibraries/{item_obj.guid}", body=body ) logger.info(f"{constants.INDENT}Active value set changed to '{active_value_set}'") else: logger.warning(f"settings.json file not found for item {item_obj.name}. Active value set not changed.") class VariableLibraryPublisher(ItemPublisher): """Publisher for Variable Library items.""" item_type = ItemType.VARIABLE_LIBRARY.value def publish_one(self, item_name: str, item: Item) -> None: """Publish a single Variable Library item.""" self.fabric_workspace_obj._publish_item(item_name=item_name, item_type=self.item_type) if not item.skip_publish: activate_value_set(self.fabric_workspace_obj, item) ================================================ FILE: src/fabric_cicd/_items/_warehouse.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Functions to process and deploy Warehouse item.""" import json import logging from fabric_cicd import constants from fabric_cicd._common._item import Item from fabric_cicd._items._base_publisher import ItemPublisher from fabric_cicd.constants import ItemType logger = logging.getLogger(__name__) class WarehousePublisher(ItemPublisher): """Publisher for Warehouse items.""" item_type = ItemType.WAREHOUSE.value def publish_one(self, item_name: str, item: Item) -> None: """Publish a single Warehouse item.""" creation_payload = next( ( json.loads(file.contents)["metadata"]["creationPayload"] for file in item.item_files if file.name == ".platform" and "creationPayload" in file.contents ), None, ) self.fabric_workspace_obj._publish_item( item_name=item_name, item_type=self.item_type, creation_payload=creation_payload, skip_publish_logging=True, ) # Check if the item is published to avoid any post publish actions if item.skip_publish: return logger.info(f"{constants.INDENT}Published Warehouse '{item_name}'") ================================================ FILE: src/fabric_cicd/_parameter/__init__.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. ================================================ FILE: src/fabric_cicd/_parameter/_parameter.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Module provides the Parameter class to load and validate the parameter file used for deployment configurations.""" import json import logging import os import re from pathlib import Path from typing import ClassVar, Optional import yaml import fabric_cicd.constants as constants from fabric_cicd._parameter._utils import ( is_valid_structure, process_input_path, replace_variables_in_parameter_file, ) # Configure logging to output to the console logger = logging.getLogger(__name__) class Parameter: """A class to validate the parameter file.""" PARAMETER_KEYS: ClassVar[dict] = { "find_replace": { "minimum": {"find_value", "replace_value"}, "maximum": {"find_value", "replace_value", "is_regex", "item_type", "item_name", "file_path"}, }, "spark_pool": { "minimum": {"instance_pool_id", "replace_value"}, "maximum": {"instance_pool_id", "replace_value", "item_name"}, }, "spark_pool_replace_value": {"type", "name"}, "key_value_replace": { "minimum": {"find_key", "replace_value"}, "maximum": {"find_key", "replace_value", "item_type", "item_name", "file_path"}, }, "gateway_binding": { "minimum": {"gateway_id", "dataset_name"}, "maximum": {"gateway_id", "dataset_name"}, }, "semantic_model_binding": { "minimum": set(), "maximum": {"connection_id", "semantic_model_name", "default", "models"}, }, "extend": {"minimum": set(), "maximum": set()}, } LOAD_ERROR_MSG = "" def __init__( self, repository_directory: Path, item_type_in_scope: list[str], environment: str, parameter_file_name: str = "parameter.yml", parameter_file_path: Optional[str] = None, ) -> None: """ Initializes the Parameter instance. Args: repository_directory: Local directory path of the repository where items are to be deployed from and parameter file lives. item_type_in_scope: Item types that should be deployed for a given workspace. environment: The environment to be used for parameterization. parameter_file_name: The name of the parameter file, default is "parameter.yml". parameter_file_path: The path to the parameter file, if not using the default. """ # Set class variables self.repository_directory = repository_directory self.item_type_in_scope = item_type_in_scope self.environment = environment self.parameter_file_name = parameter_file_name self.parameter_file_path = parameter_file_path self._set_parameter_file_path() self._refresh_parameter_file() def _set_parameter_file_path(self) -> None: """Set the parameter file path based on the provided path or default name.""" is_param_path = False original_param_path = None # Determine which input to use for parameter file path if self.parameter_file_path and isinstance(self.parameter_file_path, str): original_param_path = self.parameter_file_path if self.parameter_file_name != "parameter.yml": is_param_path = True logger.warning( constants.PARAMETER_MSGS["both_param_path_and_name"].format( self.parameter_file_name, original_param_path ) ) else: is_param_path = True try: # Resolve parameter file path, if provided if is_param_path and original_param_path: try: param_path = Path(original_param_path) # Handle relative path (must be relative to repository_directory) if not param_path.is_absolute(): logger.debug(constants.PARAMETER_MSGS["resolving_relative_path"].format(original_param_path)) param_path = Path(self.repository_directory, original_param_path) self.parameter_file_path = param_path.resolve() logger.debug(constants.PARAMETER_MSGS["using_param_file_path"].format(self.parameter_file_path)) except (TypeError, ValueError) as e: logger.error(f"Error setting parameter file path: {e}") is_param_path = False # Otherwise, resolve with default path if not is_param_path: self.parameter_file_path = Path(self.repository_directory, self.parameter_file_name).resolve() logger.debug(constants.PARAMETER_MSGS["using_default_param_file_path"].format(self.parameter_file_path)) except Exception as e: logger.error(f"Unexpected error setting parameter file path: {e}") self.parameter_file_path = None def _refresh_parameter_file(self) -> None: """Load parameters if file is present.""" self.environment_parameter = {} # Only proceed if the parameter file exists if self._validate_parameter_file_exists(): is_valid, environment_parameter = self._validate_load_parameters_to_dict() if is_valid: self.environment_parameter = environment_parameter def _validate_parameter_file_exists(self) -> bool: """Validate the parameter file exists.""" if self.parameter_file_path is None: return False return self.parameter_file_path.is_file() def _validate_load_parameters_to_dict(self) -> tuple[bool, dict]: """Validate loading the parameter file to a dictionary, including any templates.""" parameter_dict = {} try: # Load the base parameter file with Path.open(self.parameter_file_path, encoding="utf-8") as yaml_file: yaml_content = yaml_file.read() yaml_content = replace_variables_in_parameter_file(yaml_content) # Check for empty YAML content if not yaml_content.strip(): self.LOAD_ERROR_MSG = constants.PARAMETER_MSGS["invalid load"].format( constants.PARAMETER_MSGS["empty yaml"] ) return False, parameter_dict # Use custom loader that detects duplicate keys parameter_dict = yaml.load(yaml_content, Loader=_DuplicateKeyLoader) or {} logger.debug(constants.PARAMETER_MSGS["passed"].format("YAML content is valid")) if parameter_dict.get("extend"): parameter_dict = self._process_template_parameters(parameter_dict) return True, parameter_dict except (UnicodeDecodeError, yaml.YAMLError) as e: self.LOAD_ERROR_MSG = constants.PARAMETER_MSGS["invalid load"].format(e) return False, parameter_dict def _process_template_parameters(self, base_parameter_dict: dict) -> dict: """ Process template parameter files and merge them with the base parameter dictionary. Template files are resolved relative to the main parameter file's location. """ # Step 1: Check extend contains files if not isinstance(base_parameter_dict.get("extend"), list): logger.warning("No template parameter files specified under 'extend'") del base_parameter_dict["extend"] return base_parameter_dict template_files = base_parameter_dict["extend"] successful_templates = 0 failed_templates = [] processed_templates = set() # Step 2: Get the directory containing the main parameter file param_file_dir = self.parameter_file_path.parent # Step 3: Process each template file for param_file in template_files: try: # Check if this template file has been already processed to prevent duplication if param_file in processed_templates: logger.warning(f"Skipping duplicate template parameter file reference: {param_file}") continue # Step a: Resolve the path relative to the main parameter file's directory template_path = (param_file_dir / str(param_file)).resolve() # Check if the template file exists if not template_path.is_file(): error_msg = f"Template file not found: {param_file}" failed_templates.append((param_file, error_msg)) continue # Step b: Load and validate the parameter file template_dict = self._load_template_parameter_file(template_path) if not template_dict: continue # Step c: Check for nested templates if "extend" in template_dict: error_msg = f"Nested templates are not supported in {param_file}" failed_templates.append((param_file, error_msg)) continue # Step d: Merge the template dict with the base parameter dict base_parameter_dict = self._merge_template_dict(base_parameter_dict, template_dict) successful_templates += 1 # Mark template parameter file as processed processed_templates.add(param_file) logger.debug(constants.PARAMETER_MSGS["template_file_loaded"].format(template_path)) except Exception as e: error_msg = f"Error processing template file: {e!s}" failed_templates.append((param_file, error_msg)) continue # Step 4: Log results if successful_templates > 0: logger.debug(constants.PARAMETER_MSGS["template_files_processed"].format(successful_templates)) if failed_templates: for failed_file, reason in failed_templates: logger.error(f"Validation failed for template file: {failed_file}") logger.error(f"{reason}") logger.warning( f"Template parameter '{failed_file}' content will not be included in the parameter dictionary" ) elif successful_templates == 0: logger.warning(constants.PARAMETER_MSGS["template_files_none_valid"]) # Step 5: Remove the extend key after processing del base_parameter_dict["extend"] return base_parameter_dict def _load_template_parameter_file(self, file_path: Path) -> dict: """Load and validate a template parameter file.""" try: with Path.open(file_path, encoding="utf-8") as param_file: param_content = param_file.read() param_content = replace_variables_in_parameter_file(param_content) # Check for empty YAML content if not param_content.strip(): logger.error( constants.PARAMETER_MSGS["template_file_invalid"].format( file_path, constants.PARAMETER_MSGS["empty yaml"] ) ) return {} # Use custom loader that detects duplicate keys return yaml.load(param_content, Loader=_DuplicateKeyLoader) or {} except (UnicodeDecodeError, yaml.YAMLError) as e: logger.error(constants.PARAMETER_MSGS["template_file_error"].format(file_path, e)) return {} def _merge_template_dict(self, base_dict: dict, template_dict: dict) -> dict: """ Merge the template dictionary with the base dictionary, properly handling lists and nested structures. Preserves all entries, letting validation handle any issues later. """ result = base_dict.copy() for key, template_value in template_dict.items(): # Skip the 'extend' key as it's processed separately if key == "extend": continue # If the key doesn't exist in the base dict, just add it if key not in result: result[key] = template_value continue base_value = result[key] # Handle merging based on value types if isinstance(base_value, list) and isinstance(template_value, list): # For parameter lists like find_replace, append items from template result[key] = base_value + template_value elif isinstance(base_value, dict) and isinstance(template_value, dict): # For nested dictionaries, recursively merge them result[key] = self._merge_template_dict(base_value, template_value) else: # Add both values into a list for later validation result[key] = [base_value, template_value] logger.debug(f"Type mismatch for key '{key}': creating list of values for validation") return result def _validate_parameter_load(self) -> tuple[bool, str]: """Validate the parameter file load.""" if self.parameter_file_path is None: return False, "not set" if not self.environment_parameter: # Check if the file exists if not self._validate_parameter_file_exists(): logger.warning(constants.PARAMETER_MSGS["not found"].format(self.parameter_file_path)) return False, "not found" logger.debug(constants.PARAMETER_MSGS["found"]) return False, self.LOAD_ERROR_MSG return True, constants.PARAMETER_MSGS["valid load"] def _validate_parameter_file(self) -> bool: """Validate the parameter file.""" # Handle gateway_binding deprecation if "gateway_binding" in self.environment_parameter or "semantic_model_binding" in self.environment_parameter: self._handle_gateway_binding_parameter() validation_steps = [ ("parameter file load", self._validate_parameter_load), ("parameter names", self._validate_parameter_names), ("parameter file structure", self._validate_parameter_structure), ("find_replace parameter", lambda: self._validate_parameter("find_replace")), ("spark_pool parameter", lambda: self._validate_parameter("spark_pool")), ("key_value_replace parameter", lambda: self._validate_parameter("key_value_replace")), ("semantic_model_binding parameter", lambda: self._validate_parameter("semantic_model_binding")), ] for step, validation_func in validation_steps: logger.debug(constants.PARAMETER_MSGS["validating"].format(step)) is_valid, msg = validation_func() if not is_valid: # Return True for specific not is_valid case if step == "parameter file load" and msg == "not found": logger.warning(constants.PARAMETER_MSGS["terminate"].format(msg)) return True # Discontinue validation check for absent parameter if ( step in ( "find_replace parameter", "key_value_replace parameter", "spark_pool parameter", "semantic_model_binding parameter", ) and msg == "parameter not found" ): continue # Otherwise, return False with error message logger.error(constants.PARAMETER_MSGS["failed"].format(msg)) return False logger.debug(constants.PARAMETER_MSGS["passed"].format(msg)) # Return True if all validation steps pass logger.info(constants.PARAMETER_MSGS["validation_complete"]) return True def _handle_gateway_binding_parameter(self) -> None: """This code will be removed in future releases, after deprecating 'gateway_binding' support.""" # Extend semantic_model_binding by adding gateway_binding into one dict sm_param = self.environment_parameter.get("semantic_model_binding", []) gw_list = self.environment_parameter.get("gateway_binding", []) if gw_list: # Only merge gateway_binding if semantic_model_binding is legacy format (list) or doesn't exist if isinstance(sm_param, list): # Transform gateway entries to semantic_model_binding shape sm_param.extend( { "connection_id": entry.get("gateway_id"), "semantic_model_name": entry.get("dataset_name", []), } for entry in gw_list ) self.environment_parameter["semantic_model_binding"] = sm_param else: # New format (dict) - cannot merge gateway_binding, log warning logger.warning( "Cannot merge 'gateway_binding' with new format 'semantic_model_binding'. " "Please migrate gateway_binding entries to the new semantic_model_binding format." ) # Remove original gateway_binding after processing del self.environment_parameter["gateway_binding"] logger.warning(constants.PARAMETER_MSGS["gateway_deprecated"]) def _validate_parameter_structure(self) -> tuple[bool, str]: """Validate the parameter file structure.""" if not is_valid_structure(self.environment_parameter): return False, constants.PARAMETER_MSGS["invalid structure"] return True, constants.PARAMETER_MSGS["valid structure"] def _validate_parameter_names(self) -> tuple[bool, str]: """Validate the parameter names in the parameter dictionary.""" params = list(self.PARAMETER_KEYS.keys())[:6] for param in self.environment_parameter: if param not in params: return False, constants.PARAMETER_MSGS["invalid name"].format(param) return True, constants.PARAMETER_MSGS["valid name"] def _validate_parameter(self, param_name: str) -> tuple[bool, str]: """Validate the specific parameter and its contents.""" if param_name not in self.environment_parameter: logger.debug(constants.PARAMETER_MSGS["param_not_found"].format(param_name)) return False, "parameter not found" logger.debug(constants.PARAMETER_MSGS["param_found"].format(param_name)) param_count = len(self.environment_parameter[param_name]) multiple_param = param_count > 1 if multiple_param: logger.debug(constants.PARAMETER_MSGS["param_count"].format(param_count, param_name)) # Run a separate validation for semantic_model_binding parameter if param_name == "semantic_model_binding": return self._validate_semantic_model_binding_parameter(param_name, multiple_param) # Validation for other kinds of parameters validation_steps = [ ("keys", lambda param_dict: self._validate_parameter_keys(param_name, list(param_dict.keys()))), ("required values", lambda param_dict: self._validate_required_values(param_name, param_dict)), ("replace_value", lambda param_dict: self._validate_replace_value(param_name, param_dict["replace_value"])), ("optional values", lambda param_dict: self._validate_optional_values(param_name, param_dict)), ] # Set the proper find_value key name based on the parameter if param_name == "key_value_replace": key_name = "find_key" elif param_name == "spark_pool": key_name = "instance_pool_id" else: key_name = "find_value" for param_num, parameter_dict in enumerate(self.environment_parameter[param_name], start=1): param_num_str = str(param_num) if multiple_param else "" find_value = parameter_dict.get(key_name) for step, validation_func in validation_steps: logger.debug(constants.PARAMETER_MSGS["validating"].format(f"{param_name} {param_num_str} {step}")) is_valid, msg = validation_func(parameter_dict) if not is_valid: return False, msg logger.debug(constants.PARAMETER_MSGS["passed"].format(msg)) # Check if replacement will be skipped for a given find value is_valid_env, env_type = self._validate_environment(parameter_dict["replace_value"]) is_valid_optional_val, msg = self._validate_optional_values(param_name, parameter_dict, check_match=True) log_func = logger.debug if param_name == "key_value_replace" else logger.warning # Set value_type based on regex flag once value_type = ( "find value regex" if (parameter_dict.get("is_regex") and parameter_dict["is_regex"].lower() == "true") else "find value" ) if self.environment != "N/A" and not is_valid_env: if env_type.lower() == "_all_": return False, constants.PARAMETER_MSGS["other target env"].format( env_type, parameter_dict["replace_value"] ) skip_msg = constants.PARAMETER_MSGS["no target env"].format(self.environment, param_name) log_func( constants.PARAMETER_MSGS["skip"].format( value_type, find_value, skip_msg, param_name + " " + param_num_str ) ) continue if env_type.lower() == "_all_": logger.warning( constants.PARAMETER_MSGS["all target env"].format(parameter_dict["replace_value"][env_type]) ) if msg == "no match" and not is_valid_optional_val: skip_msg = constants.PARAMETER_MSGS["no filter match"] log_func( constants.PARAMETER_MSGS["skip"].format( value_type, find_value, skip_msg, param_name + " " + param_num_str ) ) return True, constants.PARAMETER_MSGS["valid parameter"].format(param_name) def _validate_semantic_model_binding_parameter( self, param_name: str, multiple_param: bool = False ) -> tuple[bool, str]: """Validate semantic_model_binding parameter, supporting both legacy and new formats. Legacy format (list): Only string connection_id allowed (no environment mapping) New format (dict): Only dict connection_id allowed (environment mapping required, use _ALL_ for all) """ param_value = self.environment_parameter.get(param_name) is_new_format = isinstance(param_value, dict) legacy_keys = {"connection_id", "semantic_model_name"} new_keys = {"default", "models"} if is_new_format: # New format: dict with 'default' and/or 'models' param_keys_set = set(param_value.keys()) # Validate top-level keys if param_keys_set & legacy_keys: return False, constants.PARAMETER_MSGS["mixed format"].format(param_name) if not param_keys_set <= new_keys: return False, constants.PARAMETER_MSGS["invalid key"].format(param_name) # Require at least one of 'default' or 'models' if "default" not in param_value and "models" not in param_value: return False, constants.PARAMETER_MSGS["missing key"].format( f"{param_name} (requires 'default' or 'models')" ) # Validate 'default' section if "default" in param_value: default_value = param_value["default"] is_valid, msg = self._validate_data_type(default_value, "dictionary", "default", param_name) if not is_valid: return False, msg if "connection_id" not in default_value: return False, constants.PARAMETER_MSGS["missing key"].format( f"{param_name}.default (requires 'connection_id')" ) # New format requires dict connection_id is_valid, msg = self._validate_connection_id( default_value["connection_id"], f"{param_name}.default.connection_id", require_dict=True ) if not is_valid: return False, msg # Validate 'models' section if "models" in param_value: models_value = param_value["models"] section_name = f"{param_name}.models" if not isinstance(models_value, list) or not models_value: return False, f"{section_name} must be a non-empty list" for idx, entry in enumerate(models_value): entry_name = f"{section_name}[{idx}]" is_valid, msg = self._validate_data_type(entry, "dictionary", entry_name, section_name) if not is_valid: return False, msg if "semantic_model_name" not in entry: return False, constants.PARAMETER_MSGS["missing key"].format( f"{entry_name} (requires 'semantic_model_name')" ) if "connection_id" not in entry: return False, constants.PARAMETER_MSGS["missing key"].format( f"{entry_name} (requires 'connection_id')" ) is_valid, msg = self._validate_data_type( entry["semantic_model_name"], "string or list[string]", "semantic_model_name", entry_name ) if not is_valid: return False, msg # New format requires dict connection_id is_valid, msg = self._validate_connection_id( entry["connection_id"], f"{entry_name}.connection_id", require_dict=True ) if not is_valid: return False, msg else: # Legacy format: list of dicts with 'connection_id' and 'semantic_model_name' for param_num, entry in enumerate(param_value, start=1): context_name = f"{param_name} {param_num}" if multiple_param else param_name param_keys_set = set(entry.keys()) # Validate keys if param_keys_set & new_keys: return False, constants.PARAMETER_MSGS["mixed format"].format(param_name) if not legacy_keys <= param_keys_set: return False, constants.PARAMETER_MSGS["missing key"].format(param_name) if not param_keys_set <= legacy_keys: return False, constants.PARAMETER_MSGS["invalid key"].format(param_name) connection_id = entry.get("connection_id") semantic_model_name = entry.get("semantic_model_name") if not connection_id: return False, constants.PARAMETER_MSGS["missing required value"].format("connection_id", param_name) if not semantic_model_name: return False, constants.PARAMETER_MSGS["missing required value"].format( "semantic_model_name", param_name ) is_valid, msg = self._validate_data_type( semantic_model_name, "string or list[string]", "semantic_model_name", param_name ) if not is_valid: return False, msg # Legacy format requires string connection_id (no environment mapping) is_valid, msg = self._validate_connection_id(connection_id, context_name, require_string=True) if not is_valid: return False, msg # Check for duplicate semantic model names self._check_duplicate_semantic_model_names(param_value, is_new_format) return True, constants.PARAMETER_MSGS["valid parameter"].format(param_name) def _validate_connection_id( self, connection_id: any, context_name: str, require_string: bool = False, require_dict: bool = False ) -> tuple[bool, str]: """Validate a connection_id value.""" # Legacy format: require string GUID only if require_string: if not isinstance(connection_id, str): return False, ( f"connection_id in {context_name} must be a string GUID. " "Environment-specific dictionaries are not supported in the legacy format. " "Please migrate to the new dictionary format with 'default' and 'models' keys." ) if not re.match(constants.VALID_GUID_REGEX, connection_id): return False, f"connection_id '{connection_id}' is not a valid GUID in {context_name}" return True, f"Valid {context_name}" # New format: require dict with environment keys if require_dict: if not isinstance(connection_id, dict): return False, ( f"{context_name} must be a dictionary with environment keys (e.g., DEV, PPE, PROD) or '_ALL_'. " "Use '_ALL_' to apply the same connection to all environments." ) if not connection_id: return False, f"{context_name} must be a non-empty dictionary" for env_key, guid_value in connection_id.items(): if not isinstance(guid_value, str): return False, f"connection_id value for environment '{env_key}' must be a string (GUID)" if not re.match(constants.VALID_GUID_REGEX, guid_value): return False, f"connection_id for environment '{env_key}' is not a valid GUID: '{guid_value}'" # Validate environment exists is_valid_env, env_type = self._validate_environment(connection_id) if self.environment != "N/A" and not is_valid_env: if env_type.lower() == "_all_": return False, constants.PARAMETER_MSGS["other target env"].format(env_type, connection_id) logger.warning(constants.PARAMETER_MSGS["no target env"].format(self.environment, context_name)) return True, f"Valid {context_name}" # Should not reach here - caller must specify require_string or require_dict return False, f"Invalid validation call for {context_name}: must specify require_string or require_dict" def _check_duplicate_semantic_model_names(self, param_value: any, is_new_format: bool) -> None: """Check for duplicate semantic model names and log a warning if found.""" names = [] if is_new_format: for item in param_value.get("models", []): raw = item.get("semantic_model_name", []) if isinstance(raw, str): names.append(raw) elif isinstance(raw, list): names.extend(n for n in raw if isinstance(n, str)) else: for entry in param_value: raw = entry.get("semantic_model_name", []) if isinstance(raw, str): names.append(raw) elif isinstance(raw, list): names.extend(n for n in raw if isinstance(n, str)) duplicates = {n for n in names if names.count(n) > 1} if duplicates: logger.warning(constants.PARAMETER_MSGS["duplicate_semantic_model"].format(", ".join(sorted(duplicates)))) def _validate_parameter_keys(self, param_name: str, param_keys: list) -> tuple[bool, str]: """Validate the keys in the parameter.""" param_keys_set = set(param_keys) # Validate minimum set if not self.PARAMETER_KEYS[param_name]["minimum"] <= param_keys_set: return False, constants.PARAMETER_MSGS["missing key"].format(param_name) # Validate maximum set if not param_keys_set <= self.PARAMETER_KEYS[param_name]["maximum"]: return False, constants.PARAMETER_MSGS["invalid key"].format(param_name) return True, constants.PARAMETER_MSGS["valid keys"].format(param_name) def _validate_required_values(self, param_name: str, param_dict: dict) -> tuple[bool, str]: """Validate required values in the parameter.""" for key in self.PARAMETER_KEYS[param_name]["minimum"]: if not param_dict.get(key): return False, constants.PARAMETER_MSGS["missing required value"].format(key, param_name) expected_type = "dictionary" if key == "replace_value" else "string" is_valid, msg = self._validate_data_type(param_dict[key], expected_type, key, param_name) if not is_valid: return False, msg # Validate find_value is a valid regex if is_regex is set to true if param_name == "find_replace": is_valid, msg = self._validate_find_regex(param_name, param_dict) if not is_valid: return False, msg if param_name == "key_value_replace": is_valid, msg = self._validate_key_value_find_key(param_dict) if not is_valid: return False, msg return True, constants.PARAMETER_MSGS["valid required values"].format(param_name) def _validate_key_value_find_key(self, param_dict: dict) -> tuple[bool, str]: """Validate the `find_key` JSONPath for key_value_replace (compile-only).""" find_key = param_dict.get("find_key") if not find_key or not isinstance(find_key, str) or not find_key.strip(): return False, "Missing or empty 'find_key' for key_value_replace" # Require absolute JSONPath root to avoid ambiguous relative paths if not find_key.strip().startswith("$"): return False, "find_key must be an absolute JSONPath starting with '$'" try: # jsonpath_ng.ext.parse supports extended JSONPath (dot/bracket) from jsonpath_ng.ext import parse parse(find_key) except Exception as e: return False, f"Invalid JSONPath expression '{find_key}': {e}" return True, "Valid JSONPath" def _validate_find_regex(self, param_name: str, param_dict: dict) -> tuple[bool, str]: """Validate the find_value is a valid regex if is_regex is set to true.""" # Return True if is_regex is not present or set if not param_dict.get("is_regex"): return True, "No regex present" # First validate is_regex value is_valid, msg = self._validate_data_type(param_dict.get("is_regex"), "string", "is_regex", param_name) if not is_valid: return False, msg # Skip regex validation if is_regex is not set to true if param_dict["is_regex"].lower() != "true": logger.warning(constants.PARAMETER_MSGS["regex_ignored"]) return True, "Skip regex validation" # Validate the find_value is a valid regex pattern = param_dict["find_value"] try: re.compile(pattern) return True, "Valid regex" except re.error as e: return False, f"Invalid regex {pattern}: {e}" def _validate_replace_value(self, param_name: str, replace_value: dict) -> tuple[bool, str]: """Validate the replace_value dictionary.""" # Validate replace_value dictionary values if param_name == "find_replace": is_valid, msg = self._validate_find_replace_replace_value(replace_value) if param_name == "key_value_replace": is_valid, msg = self._validate_key_value_replace_replace_value(replace_value) if param_name == "spark_pool": is_valid, msg = self._validate_spark_pool_replace_value(replace_value) if not is_valid: return False, msg return True, msg def _validate_find_replace_replace_value(self, replace_value: dict) -> tuple[bool, str]: """Validate the replace_value dictionary values in find_replace parameters.""" for environment in replace_value: if not replace_value[environment]: return False, constants.PARAMETER_MSGS["missing replace value"].format("find_replace", environment) is_valid, msg = self._validate_data_type( replace_value[environment], "string", environment + " replace_value", param_name="find_replace" ) if not is_valid: return False, msg return True, constants.PARAMETER_MSGS["valid replace value"].format("find_replace") def _validate_key_value_replace_replace_value(self, replace_value: dict) -> tuple[bool, str]: """Validate the replace_value dictionary values in key_value_replace parameters. For key_value_replace, we allow any data type but all values should be of the same type to ensure consistency when replacing values in JSON/YAML files. """ if not replace_value: return False, constants.PARAMETER_MSGS["missing replace value"].format("key_value_replace", "any") # Get the first value to determine the expected type first_env = next(iter(replace_value)) first_value = replace_value[first_env] expected_type = type(first_value) for environment in replace_value: value = replace_value[environment] # Check if value is None/empty (not allowed) if value is None: return False, constants.PARAMETER_MSGS["missing replace value"].format("key_value_replace", environment) # Check type consistency across all environments if type(value) != expected_type: return ( False, f"Inconsistent data types in key_value_replace replace_value: " f"'{first_env}' has type {expected_type.__name__} but " f"'{environment}' has type {type(value).__name__}. " f"All values must be of the same type.", ) return True, constants.PARAMETER_MSGS["valid replace value"].format("key_value_replace") def _validate_spark_pool_replace_value(self, replace_value: dict) -> tuple[bool, str]: """Validate the replace_value dictionary values in spark_pool parameter.""" for environment, environment_dict in replace_value.items(): # Check if environment_dict is empty if not environment_dict: return False, constants.PARAMETER_MSGS["missing replace value"].format("spark_pool", environment) is_valid, msg = self._validate_data_type( environment_dict, "dictionary", environment + " key", param_name="spark_pool" ) if not is_valid: return False, msg msgs = constants.PARAMETER_MSGS["invalid replace value"] # Validate keys for the environment config_keys = list(environment_dict.keys()) required_keys = self.PARAMETER_KEYS["spark_pool_replace_value"] if not required_keys.issubset(config_keys) or len(config_keys) != len(required_keys): return False, msgs["missing key"].format(environment) # Validate values for the environment dict for key in config_keys: if not environment_dict[key]: return False, msgs["missing value"].format(environment, key) is_valid, msg = self._validate_data_type(environment_dict[key], "string", key, param_name="spark_pool") if not is_valid: return False, msg if environment_dict["type"] not in ["Capacity", "Workspace"]: return False, msgs["invalid value"].format(environment) return True, constants.PARAMETER_MSGS["valid replace value"].format("spark_pool") def _validate_optional_values( self, param_name: str, param_dict: dict, check_match: bool = False ) -> tuple[bool, str]: """Validate the optional filter values in the parameter.""" optional_values = { "item_type": param_dict.get("item_type"), "item_name": param_dict.get("item_name"), "file_path": param_dict.get("file_path"), } if (param_name == "find_replace" and not any(optional_values.values())) or ( param_name == "spark_pool" and not optional_values["item_name"] ): return True, constants.PARAMETER_MSGS["no optional"].format(param_name) validation_methods = { "item_type": self._validate_item_type, "item_name": self._validate_item_name, "file_path": self._validate_file_path, } for param, value in optional_values.items(): if value: # Check value data type is_valid, msg = self._validate_data_type(value, "string or list[string]", param, param_name) if not is_valid: return False, msg # Validate specific optional values and check for matches if check_match and param in validation_methods: values = value if isinstance(value, list) else [value] if param == "file_path": is_valid, msg = validation_methods[param](values) else: for item in values: is_valid, msg = validation_methods[param](item) if not is_valid: logger.error(msg) return False, "no match" return True, constants.PARAMETER_MSGS["valid optional"].format(param_name) def _validate_data_type( self, input_value: any, expected_type: str, input_name: str, param_name: str ) -> tuple[bool, str]: """Validate the data type of the input value.""" type_validators = { "string": lambda x: isinstance(x, str), "string or list[string]": lambda x: (isinstance(x, str)) or (isinstance(x, list) and all(isinstance(item, str) for item in x)), "dictionary": lambda x: isinstance(x, dict), } # Check if the expected type is valid and if the input matches the expected type if expected_type not in type_validators or not type_validators[expected_type](input_value): return False, constants.PARAMETER_MSGS["invalid data type"].format(input_name, expected_type, param_name) return True, "Data type is valid" def _validate_environment(self, replace_value: dict) -> tuple[bool, str]: """ Check the target environment exists as a key in the replace_value dictionary. If "_ALL_" (case insensitive) is present, it must be the only key. """ # Check for _ALL_ in any case variation all_key = None for key in replace_value: if key.lower() == "_all_": logger.warning(f"Found the reserved environment key '{key}'") all_key = key break if all_key: # If _ALL_ is present, it must be the only key return len(replace_value) == 1, all_key # If _ALL_ is not present, check if target environment is present return self.environment in replace_value, "env" def _validate_item_type(self, input_type: str) -> tuple[bool, str]: """Validate the item type is in scope.""" if input_type not in self.item_type_in_scope: return False, constants.PARAMETER_MSGS["invalid item type"].format(input_type) return True, "Valid item type" def _validate_item_name(self, input_name: str) -> tuple[bool, str]: """Validate the item name is found in the repository directory.""" item_name_list = [] for root, _dirs, files in os.walk(self.repository_directory): directory = Path(root) # valid item directory with .platform file within if ".platform" in files: item_metadata_path = Path(directory, ".platform") with Path.open(item_metadata_path, encoding="utf-8") as file: item_metadata = json.load(file) # Ensure required metadata fields are present if item_metadata and "type" in item_metadata["metadata"] and "displayName" in item_metadata["metadata"]: item_name = item_metadata["metadata"]["displayName"] item_name_list.append(item_name) # Check if item name is valid if input_name not in item_name_list: return False, constants.PARAMETER_MSGS["invalid item name"].format(input_name) return True, "Valid item name" def _validate_file_path(self, input_path: list[str]) -> tuple[bool, str]: """Validate that the file paths exist within the repository directory.""" # Convert input path to Path objects, returned as a list of valid paths valid_paths = process_input_path(self.repository_directory, input_path, validation_flag=True) # If list of paths is empty, all paths were invalid if not valid_paths: return False, constants.PARAMETER_MSGS["no valid file path"].format(input_path) # Check for some invalid paths path_diff = len(input_path) - len(valid_paths) if path_diff > 0: return False, constants.PARAMETER_MSGS["invalid file path"].format(input_path, path_diff) return True, "Valid file path" class _DuplicateKeyLoader(yaml.SafeLoader): """Custom YAML loader that raises an error on duplicate keys.""" pass def _collect_duplicate_key_errors(root_node: yaml.MappingNode, loader: _DuplicateKeyLoader) -> list[str]: """Collect duplicate key errors from all mapping nodes using iterative traversal.""" errors: list[str] = [] stack = [root_node] while stack: node = stack.pop() # Group original key names by their lowercase form seen_keys: dict[str, list[str]] = {} for key_node, value_node in node.value: key = loader.construct_object(key_node) # Case-insensitive comparison (e.g., "PROD" and "prod" are duplicates) normalized_key = key.lower() if isinstance(key, str) else key if normalized_key not in seen_keys: seen_keys[normalized_key] = [] seen_keys[normalized_key].append(str(key)) # Queue child mappings for checking at their own level if isinstance(value_node, yaml.MappingNode): stack.append(value_node) elif isinstance(value_node, yaml.SequenceNode): for item_node in value_node.value: if isinstance(item_node, yaml.MappingNode): stack.append(item_node) for _, variants in seen_keys.items(): if len(variants) > 1: unique_variants = set(variants) count = len(variants) # Show key once if all cases match, otherwise show all variants if len(unique_variants) == 1: errors.append(f"'{next(iter(unique_variants))}' ({count})") else: errors.append(f"{sorted(unique_variants)} ({count})") return errors def _check_duplicate_keys_constructor(loader: _DuplicateKeyLoader, node: yaml.MappingNode) -> dict: """Constructor that checks for duplicate keys in YAML mappings.""" # Run full traversal only from the root node; inner nodes skip to avoid redundant checks if not getattr(loader, "_root_checked", False): loader._root_checked = True errors = _collect_duplicate_key_errors(node, loader) if errors: dupe_details = ", ".join(errors) raise yaml.YAMLError(constants.PARAMETER_MSGS["duplicate key"].format(dupe_details)) return loader.construct_mapping(node) _DuplicateKeyLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _check_duplicate_keys_constructor) ================================================ FILE: src/fabric_cicd/_parameter/_utils.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """ Following functions are parameter utilities used by the FabricWorkspace and Parameter classes, and for debugging the parameter file. The utilities include validating the parameter.yml file, determining parameter dictionary structure, processing parameter values, and handling parameter value replacements. """ import glob import json import logging import os import re import urllib.parse from pathlib import Path from typing import Optional, Union import yaml from jsonpath_ng.ext import parse import fabric_cicd.constants as constants from fabric_cicd import FabricWorkspace from fabric_cicd._common._exceptions import InputError, ParsingError from fabric_cicd.constants import ItemType logger = logging.getLogger(__name__) """Functions to extract parameter values""" def _validate_regex_structure(pattern: re.Pattern, find_value: str) -> None: """ Validates regex pattern structure to ensure it has exactly one capturing group. This validation is performed independently of whether the pattern matches any content. Args: pattern: Compiled regex pattern find_value: The regex pattern string for error messages Raises: InputError: If the regex doesn't have exactly one capturing group """ # Check the number of capturing groups in the pattern if pattern.groups != 1: msg = f"Regex pattern '{find_value}' must contain exactly one capturing group." raise InputError(msg, logger) def _validate_regex_pattern(matches: list, find_value: str) -> None: """ Validates regex pattern matches to ensure the captured value is not empty. Args: matches: List of regex match objects find_value: The regex pattern string for error messages Raises: InputError: If validation fails """ if matches: # Check if the captured group is empty (which would be invalid) captured_value = matches[0].group(1) if not captured_value: msg = f"Regex pattern '{find_value}' captured an empty value." raise InputError(msg, logger) def extract_find_value(param_dict: dict, file_content: str, filter_match: bool) -> dict: """ Extracts the find_value and sets the value. Processes the find_value if a valid regex is provided. Returns replacement information for use with re.sub() or string replace(). Args: param_dict: The parameter dictionary containing the find_value and is_regex keys. file_content: The content of the file where the find_value will be searched. filter_match: A boolean to check for a regex match in filtered files only. Returns: Dictionary with keys: - 'pattern': The find pattern (original string or regex pattern) - 'is_regex': Whether this is a regex pattern - 'has_matches': Whether any matches were found """ find_value = param_dict.get("find_value") is_regex = param_dict.get("is_regex", "").lower() == "true" # No find value -> nothing to do if not find_value: return {"pattern": "", "is_regex": False, "has_matches": False} # Regex find_value if is_regex: try: compiled = re.compile(find_value) except re.error as re_err: msg = f"Invalid regex '{find_value}': {re_err}" raise InputError(msg, logger) from re_err # Validate structure (to catch bad patterns early) _validate_regex_structure(compiled, find_value) # If file excluded by filters, do not search — return no-match but keep validation if not filter_match: return {"pattern": find_value, "is_regex": True, "has_matches": False} matches = list(re.finditer(compiled, file_content)) _validate_regex_pattern(matches, find_value) return {"pattern": find_value, "is_regex": True, "has_matches": bool(matches)} # Non-regex find_value if not filter_match: return {"pattern": find_value, "is_regex": False, "has_matches": False} return {"pattern": find_value, "is_regex": False, "has_matches": find_value in file_content} def extract_replace_value(workspace_obj: FabricWorkspace, replace_value: str, get_dataflow_name: bool = False) -> str: """Extracts the replace_value and sets the value. Processes the replace_value if a valid variable is provided.""" if not replace_value.startswith("$"): if get_dataflow_name: logger.debug( "Can't get dataflow name as the replace_value was set to a regular string, not the items variable" ) return None return replace_value # If $workspace variable, return the workspace ID value if replace_value.startswith("$workspace."): if get_dataflow_name: msg = "Invalid replace_value variable: '$workspace'. Expected format to get dataflow name: $items.type.name.$attribute" raise InputError(msg, logger) return _extract_workspace_id(workspace_obj, replace_value) # If $items variable, return the item attribute value if found if replace_value.startswith("$items."): return _extract_item_attribute(workspace_obj, replace_value, get_dataflow_name) # Otherwise, raise an error for invalid variable syntax msg = f"Invalid replace_value variable format: '{replace_value}'. Expected format: $items.type.name.$attribute or $workspace.$id or $workspace.$name or $workspace.$name_encoded or $workspace..$id or $workspace..$items...$attribute" raise InputError(msg, logger) def _extract_workspace_id(workspace_obj: FabricWorkspace, replace_value: str) -> str: """ Extracts workspace ID or display name from the $workspace variable to set as the replace_value. Supports the following formats: - $workspace.id or $workspace.$id - Returns the target workspace ID - $workspace.$name - Returns the target workspace display name - $workspace.$name_encoded - Returns the target workspace display name, URL-encoded - $workspace..$items...$ - Resolves an item attribute from the specified workspace, where $attribute is any supported attribute in constants.ITEM_ATTR_LOOKUP - $workspace. or $workspace..$id - Resolves the workspace ID from the name """ # Case 1: $workspace.$id ($workspace.id supported for backward compatibility) if replace_value == "$workspace.id" or replace_value == "$workspace.$id": return workspace_obj.workspace_id try: # Case 2: $workspace.$name if replace_value == "$workspace.$name": return workspace_obj._resolve_workspace_name() # Case 3: $workspace.$name_encoded - URL-encoded display name if replace_value == "$workspace.$name_encoded": name = workspace_obj._resolve_workspace_name() return urllib.parse.quote(name, safe="") # Extract the variable string without the prefix var_string = replace_value.removeprefix("$workspace.") # Case 4: Check if this is a cross-workspace item reference if "$items." in var_string: # Check if the variable ends with a valid attribute valid_attribute = False attribute = None for attr in constants.ITEM_ATTR_LOOKUP: if var_string.endswith(f".${attr}"): valid_attribute = True attribute = attr break if not valid_attribute: msg = f"Invalid syntax or missing attribute in cross-workspace variable '{replace_value}'. Expected format: $workspace.name.$items.type.name.$attribute where attribute is one of: {', '.join(constants.ITEM_ATTR_LOOKUP)}" raise ParsingError(msg, logger) # Split on the $items prefix to get workspace name workspace_part, items_part = var_string.split(".$items.", 1) workspace_name = workspace_part.strip() logger.debug(f"Extracted workspace name: {workspace_name}") # Get workspace ID workspace_id = workspace_obj._resolve_workspace_id(workspace_name) # Remove the trailing .$attribute to get the item info items_info = items_part.removesuffix(f".${attribute}") # Find the last period to separate item type from item name last_period_pos = items_info.rfind(".") if last_period_pos == -1: msg = f"Invalid $workspace variable syntax: {replace_value}. Expected format: $workspace.name.$items.type.name.$attribute" raise ParsingError(msg, logger) # Extract item_type and item_name item_type = items_info[:last_period_pos].strip() item_name = items_info[last_period_pos + 1 :].strip() logger.debug(f"Extracted item type: {item_type}, item name: {item_name}, attribute: {attribute}") if item_type not in constants.ACCEPTED_ITEM_TYPES: msg = f"Item type '{item_type}' is invalid or not supported" raise ParsingError(msg, logger) # Look up the attribute value of the item in the specified workspace attribute_value = workspace_obj._lookup_item_attribute(workspace_id, item_type, item_name, attribute) logger.debug(f"Found item {attribute}: {attribute_value}") return attribute_value # Case 5: Pattern: $workspace..$id (explicit) or $workspace. (backward-compatible) workspace_name = var_string.removesuffix(".$id").strip() logger.debug(f"Extracted workspace name: {workspace_name}") # Resolve workspace ID return workspace_obj._resolve_workspace_id(workspace_name) except Exception as e: # Re-raise exceptions if isinstance(e, (ParsingError, InputError)): raise e # Otherwise, wrap it in a ParsingError msg = f"Error parsing $workspace variable: {e}" raise ParsingError(msg, logger) from e def _extract_item_attribute(workspace_obj: FabricWorkspace, variable: str, get_dataflow_name: bool) -> str: """ Extracts the item attribute value from the $items variable to set as the replace_value. Args: workspace_obj: The FabricWorkspace object containing the workspace items dictionary used to access item metadata. variable: The $items variable string to be parsed and processed. Supports the following formats: - $items.type.name.attribute (legacy format) - $items.type.name.$attribute (new format) Supported attributes: id, sqlendpoint, queryserviceuri get_dataflow_name: A boolean flag to indicate if the dataflow item name should be returned instead of the attribute value. """ error = None try: var_string = variable.removeprefix("$items.") # Check for new pattern with $attribute if ".$" in var_string: # Split on the $attribute marker name_part, attr_part = var_string.rsplit(".$", 1) # Extract attribute name attribute = attr_part.strip() # Find the last period to separate item_type from item_name last_period_pos = name_part.rfind(".") if last_period_pos == -1: msg = f"Invalid $items variable syntax: {variable}. Expected format: $items.type.name.$attribute" error = ParsingError(msg, logger) return None # Extract item_type and item_name item_type = name_part[:last_period_pos].strip() item_name = name_part[last_period_pos + 1 :].strip() # Backward compatibility for legacy pattern else: msg = f"Invalid $items variable syntax: {variable}. Expected format: $items.type.name.attribute" # Split the string to get item_type (first part) parts = var_string.split(".", 1) if len(parts) < 2: error = ParsingError(msg, logger) return None item_type = parts[0].strip() # Get the attribute (last part) if "." not in parts[1]: error = ParsingError(msg, logger) return None # Find the position of the last period which separates item_name from attribute last_period_pos = parts[1].rfind(".") if last_period_pos == -1: error = ParsingError(msg, logger) return None # Extract item_name and attribute item_name = parts[1][:last_period_pos].strip() attribute = parts[1][last_period_pos + 1 :].strip() # Validate attribute before further processing attr_name = attribute.lower() if attr_name not in constants.ITEM_ATTR_LOOKUP: msg = f"Attribute '{attribute}' is invalid. Supported attributes: {list(constants.ITEM_ATTR_LOOKUP)}" error = ParsingError(msg, logger) return None logger.debug( f"Processing $items variable with item_type={item_type}, item_name={item_name}, attribute={attribute}" ) # Refresh the workspace items to get the latest deployed items workspace_obj._refresh_deployed_items() # Validate item type exists in the deployed workspace if item_type not in workspace_obj.workspace_items and not get_dataflow_name: msg = f"Item type '{item_type}' is invalid or not found in deployed items" error = ParsingError(msg, logger) return None # Check if the specific item is deployed if item_name not in workspace_obj.workspace_items.get(item_type, {}) and not get_dataflow_name: msg = f"Item '{item_name}' not found as a deployed {item_type}" error = ParsingError(msg, logger) return None # Special case: set to True in the context of a Dataflow that references another Dataflow if get_dataflow_name: if ( item_type in workspace_obj.repository_items and item_type == ItemType.DATAFLOW.value and item_name in workspace_obj.repository_items[item_type] and attribute == "id" ): logger.debug("Source Dataflow reference will be replaced separately") return item_name # Return None for non-existent item return None # Get the item's attributes from workspace items item_attr_values = workspace_obj.workspace_items[item_type][item_name] # Get the attribute value and check if it exists attr_value = item_attr_values.get(attr_name) if not attr_value: msg = f"Value does not exist for attribute '{attribute}' in the {item_type} item '{item_name}'" error = ParsingError(msg, logger) return None logger.debug(f"Found attribute '{attr_name}' with value '{attr_value}'") return attr_value except Exception as e: # If it's not a ParsingError, create a new one if not isinstance(e, ParsingError): error = ParsingError(f"Error parsing $items variable: {e!s}", logger) error = e return None finally: # Raise error at the very end (only once) if error is not None: raise error def extract_parameter_filters(workspace_obj: FabricWorkspace, param_dict: dict) -> tuple[str, str, Path]: """Extracts the item type, name, and path filters from the parameter dictionary, if present.""" item_type = param_dict.get("item_type") item_name = param_dict.get("item_name") file_path = process_input_path(workspace_obj.repository_directory, param_dict.get("file_path")) return item_type, item_name, file_path def process_environment_key(environment: str, replace_value_dict: dict) -> dict: """Processes the replace_value dictionary to replace the '_ALL_' environment key with the target environment when present.""" # If there's only one key, check if it's "_ALL_" (case insensitive) and replace it # Note: When other env keys are present with _ALL_, upstream parameter validation fails if len(replace_value_dict) == 1: key = next(iter(replace_value_dict)) if key.lower() == "_all_": replace_value_dict[environment] = replace_value_dict.pop(key) return replace_value_dict """Functions to replace key values in JSON or YAML""" def replace_key_value( workspace_obj: FabricWorkspace, param_dict: dict, content: str, env: str, is_yaml: bool = False ) -> str: """A function to replace key values in a JSON or YAML using parameterization. It uses jsonpath_ng to find and replace values in the JSON. Args: workspace_obj: The FabricWorkspace object. param_dict: The parameter dictionary. content: the JSON/YAML content to be modified. env: The environment variable to be used for replacement. is_yaml: A boolean indicating if the content is YAML (default is False for JSON). """ # Parse content to a dictionary based on format (YAML or JSON) if is_yaml: try: data = yaml.safe_load(content) except yaml.YAMLError as ye: raise ValueError(ye) from ye # Handle empty YAML files if data is None: return content else: try: data = json.loads(content) except json.JSONDecodeError as jde: raise ValueError(jde) from jde # Extract the jsonpath expression from the find_key attribute of the param_dict jsonpath_expr = parse(param_dict["find_key"]) replace_value_dict = process_environment_key(workspace_obj.environment, param_dict["replace_value"]) for match in jsonpath_expr.find(data): # If the env is present in the replace_value array perform the replacement if env in replace_value_dict: try: # Process the replace value to handle $items notation processed_value = replace_value_dict[env] if isinstance(processed_value, str): processed_value = extract_replace_value(workspace_obj, processed_value) match.full_path.update(data, processed_value) logger.debug( f"Value: {match.value} found at path: {match.full_path} to be replaced with value: {processed_value}" ) except Exception as match_e: raise ValueError(match_e) from match_e return yaml.dump(data, default_flow_style=False, allow_unicode=True) if is_yaml else json.dumps(data) def replace_variables_in_parameter_file(raw_file: str) -> str: """ A function to replace tokens in the parameter.yml file with environment variables. Args: raw_file: The parameter.yml file content as a string. """ if "enable_environment_variable_replacement" in constants.FEATURE_FLAG: # filter os.environ dict to only allow variables that begin with $ENV: env_vars = {k[len("$ENV:") :]: v for k, v in os.environ.items() if k.startswith("$ENV:")} # block of code to support both variants of the parameters.yml file # Perform replacements for var_name, var_value in env_vars.items(): placeholder = f"$ENV:{var_name}" if placeholder in raw_file: raw_file = raw_file.replace(placeholder, var_value) logger.debug(f"Replaced {placeholder} with {var_value}") return raw_file return raw_file """Functions to validate the parameter file""" def validate_parameter_file( repository_directory: str, item_type_in_scope: Optional[list] = None, environment: str = "N/A", parameter_file_name: str = "parameter.yml", parameter_file_path: Optional[str] = None, ) -> bool: """ A wrapper function that validates a parameter.yml file, using the Parameter class. Args: repository_directory: The directory containing the items and parameter.yml file. item_type_in_scope: A list of item types to validate. If omitted, defaults to all supported item types. environment: The target environment. parameter_file_name: The name of the parameter file, default is "parameter.yml". parameter_file_path: The path to the parameter file, if different from the default. """ from fabric_cicd._common._validate_input import ( validate_environment, validate_item_type_in_scope, validate_repository_directory, ) # Import the Parameter class here to avoid circular imports from fabric_cicd._parameter._parameter import Parameter # Initialize the Parameter object with the validated inputs parameter_obj = Parameter( repository_directory=validate_repository_directory(repository_directory), item_type_in_scope=validate_item_type_in_scope(item_type_in_scope), environment=validate_environment(environment), parameter_file_name=parameter_file_name, parameter_file_path=parameter_file_path, ) # Validate with _validate_parameter_file() method return parameter_obj._validate_parameter_file() def is_valid_structure(param_dict: dict, param_name: Optional[str] = None) -> bool: """ Checks the parameter dictionary structure and determines if it contains the valid structure (i.e. a list of values when indexed by the key, or the new dict format for semantic_model_binding). Args: param_dict: The parameter dictionary to check. param_name: The name of the parameter to check, if specified. """ # Check the structure of the specified parameter if param_name: # Special case for semantic_model_binding - can be list (legacy) or dict (new) if param_name == "semantic_model_binding": is_valid, _ = _check_semantic_model_binding_structure(param_dict.get(param_name)) return is_valid return _check_parameter_structure(param_dict.get(param_name)) # Get only parameters that exist in param_dict existing_params = [name for name in constants.PARAM_NAMES if name in param_dict] # If no parameters found, return False if not existing_params: return False # Check all existing parameters have valid structure for name in existing_params: # Special case for semantic_model_binding if name == "semantic_model_binding": is_valid, _ = _check_semantic_model_binding_structure(param_dict.get(name)) if not is_valid: return False elif not _check_parameter_structure(param_dict.get(name)): return False return True def _check_parameter_structure(param_value: any) -> bool: """Checks the structure of a parameter value""" return isinstance(param_value, list) def _check_semantic_model_binding_structure(param_value: any) -> tuple[bool, bool]: """ Checks the structure of semantic_model_binding parameter value. Supports both legacy (list) and new (dict with 'default' or 'models') formats. """ # Legacy format: list if isinstance(param_value, list): return (True, False) # New format: dict with at least 'default' or 'models' if isinstance(param_value, dict): has_default = "default" in param_value has_models = "models" in param_value is_valid = has_default or has_models return (is_valid, True) return (False, False) """Functions to process and validate file paths from the optional filter""" def process_input_path( repository_directory: Path, input_path: Union[str, list[str], None], validation_flag: bool = False ) -> Union[list[Path], None]: """ Processes the input_path value according to its type. Supports both regular paths and wildcard paths, including mixed lists. Args: repository_directory: The directory of the repository. input_path: The input path value to process (None, a string, or list of strings). validation_flag: Flag to indicate the context of the function call to set the logging type. """ # Set the logging function based on validation_flag log_func = logger.error if validation_flag else logger.debug # Return None for None or empty input if not input_path: return None # Use a set to avoid duplicate paths valid_paths = set() # Standardize to list for consistent processing paths_to_process = [input_path] if isinstance(input_path, str) else input_path for path in paths_to_process: # Process path based on whether it contains wildcard characters has_wildcard = False try: has_wildcard = glob.has_magic(path) except Exception as e: log_func(f"Error checking for wildcard in path '{path}': {e}") continue if has_wildcard: _process_wildcard_path(path, repository_directory, valid_paths, log_func) else: _process_regular_path(path, repository_directory, valid_paths, log_func) return list(valid_paths) def _process_regular_path( path: str, repository_directory: Path, valid_paths: set[Path], log_func: logging.Logger ) -> None: """Process a regular (non-wildcard) path and add to valid_paths if valid.""" # Normalize path for consistent handling normalized_path = Path(path.lstrip("/\\")) # Set the path type based on whether it is absolute or relative path_type = "Relative" if not normalized_path.is_absolute() else "Absolute" # Validate the path and add to set if valid valid_path = _resolve_file_path(normalized_path, repository_directory, path_type, log_func) if valid_path: valid_paths.add(valid_path) def _process_wildcard_path( path: str, repository_directory: Path, valid_paths: set[Path], log_func: logging.Logger ) -> None: """Process a wildcard path and add matching files to valid_paths if valid.""" search_pattern = _set_wildcard_path_pattern(path, repository_directory, log_func) if not search_pattern: return # Track if matches are found initial_paths_count = len(valid_paths) # Get all matching files that exist try: for matched_path in [p for p in repository_directory.glob(search_pattern) if p.is_file()]: # Validate path and add to set if valid valid_path = _resolve_file_path(matched_path, repository_directory, "Wildcard", log_func) if valid_path: valid_paths.add(valid_path) # Only log if matches were not found if len(valid_paths) == initial_paths_count: log_func(f"Wildcard path '{path}' did not match any files") except Exception as e: log_func(f"Error processing wildcard pattern '{search_pattern}': {e}") def _set_wildcard_path_pattern(wildcard_path: str, repository_directory: Path, log_func: logging.Logger) -> str: """Determine the glob search pattern for a wildcard path.""" normalized_wildcard_path = wildcard_path.replace("\\", "/") # Step 1: Validate wildcard pattern syntax if not _validate_wildcard_syntax(normalized_wildcard_path, log_func): return "" # Step 2: Determine search pattern based on path type if normalized_wildcard_path.startswith("**/"): logger.debug("Recursive wildcard path detected") return f"**/{normalized_wildcard_path[3:]}" if Path(normalized_wildcard_path).is_absolute(): logger.debug("Absolute wildcard path detected") try: # Check if the path is within the repository rel_path = Path(normalized_wildcard_path).relative_to(repository_directory) return str(rel_path) except ValueError: log_func(f"Invalid absolute wildcard path. '{wildcard_path}' is outside the repository directory") return "" else: logger.debug("Non-recursive and non-absolute wildcard path detected") return normalized_wildcard_path def _resolve_file_path( input_path: Path, repository_directory: Path, path_type: str, log_func: logging.Logger ) -> Optional[Path]: """ Validates that a path exists, is a file, and is within the repository directory. Returns the resolved absolute path if valid, None otherwise. """ try: # Step 1: Resolve the input path based on its type if path_type == "Relative": resolved_path = (repository_directory / input_path).resolve() logger.debug(f"{path_type} path '{input_path}' resolved as '{resolved_path}'") elif path_type == "Absolute": resolved_path = input_path.resolve() else: resolved_path = input_path # Step 2: Check if the path is within the repository directory try: _ = resolved_path.relative_to(repository_directory) except ValueError: log_func(f"{path_type} path '{input_path}' is outside the repository directory") return None # Step 3: For non-wildcard paths, check existence and file type if path_type != "Wildcard": # Check path existence if not resolved_path.exists(): log_func(f"{path_type} path '{input_path}' does not exist") return None # Check file validation if not resolved_path.is_file(): log_func(f"{path_type} path '{input_path}' is not a file") return None logger.debug(f"Path '{resolved_path}' is valid and within the repository directory") return resolved_path except Exception as e: log_func(f"Error validating {path_type.lower()} path '{input_path}': {e}") return None def _validate_wildcard_syntax(pattern: str, log_func: logging.Logger) -> bool: """Validates wildcard pattern syntax before using glob.""" # Check for empty or whitespace-only patterns if not pattern or pattern.isspace(): log_func("Wildcard pattern is empty") return False # Check for problematic absolute paths with recursive patterns if pattern.startswith("/") and pattern[1:].startswith("**/"): log_func(f"Absolute path with recursive pattern is not allowed: '{pattern}'") return False # Handle Windows-style absolute paths with recursive patterns if re.match(r"^[a-zA-Z]:\\", pattern) and "**\\" in pattern: log_func(f"Absolute path with recursive pattern is not allowed: '{pattern}'") return False # Apply standard validations from constants for validation in constants.WILDCARD_PATH_VALIDATIONS: if validation["check"](pattern): log_func(validation["message"](pattern)) return False # Validate proper nesting of brackets and braces if not _validate_nested_brackets_braces(pattern, log_func): return False # Validate character classes (bracket expressions) for section in re.findall(r"\[(.*?)\]", pattern): if not section or section.startswith("]") or section.startswith("-") or "--" in section: log_func(f"Invalid character class in pattern: '{pattern}'") return False # Validate brace expansions try: for section in re.findall(r"\{(.*?)\}", pattern): if ( not section # Empty braces or "," not in section # No comma separator or section.startswith(",") # Starts with comma or section.endswith(",") # Ends with comma or ",," in section ): # Adjacent commas log_func(f"Invalid brace expansion in pattern: '{pattern}'") return False except Exception as e: log_func(f"Error validating brace content in pattern '{pattern}': {e}") return False # Check for path traversal sequences pattern_lower = pattern.lower() traversal_patterns = [ "../", ".." + os.sep, ".." + os.altsep if os.altsep else "", "..%2F", "..%5C", "..%2f", "..%5c", ] for traversal in traversal_patterns: if traversal and traversal in pattern_lower: log_func(f"Path traversal sequences not allowed: '{pattern}'") return False return True def _validate_nested_brackets_braces(pattern: str, log_func: logging.Logger) -> bool: """Validates proper nesting of brackets and braces in a wildcard pattern.""" stack = [] for pos, char in enumerate(pattern): if char in "[{": stack.append(char) elif char in "]}": # Check if stack is empty (closing without opening) if not stack: log_func(f"Unmatched closing '{char}' at position {pos} in pattern: '{pattern}'") return False # Check for proper matching last_open = stack.pop() if (char == "]" and last_open != "[") or (char == "}" and last_open != "{"): log_func(f"Mismatched bracket/brace pair '{last_open}{char}' at position {pos} in pattern: '{pattern}'") return False # Check if all brackets and braces were closed if stack: log_func(f"Unclosed bracket(s) or brace(s) in pattern: '{pattern}'") return False return True """Functions to determine replacement based on optional filters""" def check_replacement( input_type: Union[str, list[str], None], input_name: Union[str, list[str], None], input_path: Union[list[Path], None], item_type: str, item_name: str, file_path: Path, ) -> bool: """ Determines whether a find and replace is applied or not based on the provided optional filters. Args: input_type: The input item_type value to check. input_name: The input item_name value to check. input_path: The input file_path value to check. item_type: The item_type value to compare with. item_name: The item_name value to compare with. file_path: The file_path value to compare with. """ # No optional parameters found if input_type is None and input_name is None and input_path is None: logger.debug("No optional filters found. Find and replace applied in this repository file") return True # Otherwise, find matches for the optional parameters item_type_match = _find_match(input_type, item_type) item_name_match = _find_match(input_name, item_name) file_path_match = _find_match(input_path, file_path) if item_type_match and item_name_match and file_path_match: if input_type: logger.debug(f"Item type match found: {item_type_match}") if input_name: logger.debug(f"Item name match found: {item_name_match}") if input_path: logger.debug(f"File path match found: {file_path_match}") # Optional filters match found. Find and replace applied in this repository file return True # Optional filters match not found. Find and replace skipped for this repository file return False def _find_match( param_value: Union[str, list, None], compare_value: Union[str, Path], ) -> bool: """ Checks for a match between the parameter value and the compare value based on parameter value type. Args: param_value: The parameter value to compare (can be a string, list, Path, or None type). compare_value: The value to compare with. """ # If no parameter value, checking for matches is not required if param_value is None: return True # Otherwise, check for matches based on the parameter value type if isinstance(param_value, list): match_condition = any(compare_value == value for value in param_value) elif isinstance(param_value, (str, Path)): match_condition = compare_value == param_value else: match_condition = False return match_condition ================================================ FILE: src/fabric_cicd/constants.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Constants for the fabric-cicd package.""" import os from enum import Enum from fabric_cicd._common._validate_env_vars import VALID_GUID_REGEX as VALID_GUID_REGEX from fabric_cicd._common._validate_env_vars import validate_env_var_api_url # General VERSION = "1.0.0" DEFAULT_GUID = "00000000-0000-0000-0000-000000000000" FEATURE_FLAG = set() USER_AGENT = f"ms-fabric-cicd/{VERSION}" VALID_ENABLE_FLAGS = ["1", "true", "yes"] class EnvVar(str, Enum): """Enumeration of environment variables used by fabric-cicd.""" HTTP_TRACE_ENABLED = "FABRIC_CICD_HTTP_TRACE_ENABLED" """Set to '1', 'true', or 'yes' to enable HTTP request/response tracing.""" HTTP_TRACE_FILE = "FABRIC_CICD_HTTP_TRACE_FILE" """Path to save HTTP trace output. Only used if HTTP tracing is enabled.""" DEFAULT_API_ROOT_URL = "DEFAULT_API_ROOT_URL" """Override the default Power BI API root URL. Defaults to 'https://api.powerbi.com'.""" FABRIC_API_ROOT_URL = "FABRIC_API_ROOT_URL" """Override the Fabric API root URL. Defaults to 'https://api.fabric.microsoft.com'.""" RETRY_DELAY_OVERRIDE_SECONDS = "FABRIC_CICD_RETRY_DELAY_OVERRIDE_SECONDS" """Override retry delay in seconds (e.g., '0' for instant retries - useful in tests).""" RETRY_AFTER_SECONDS = "FABRIC_CICD_RETRY_AFTER_SECONDS" """Override retry-after delay for item name conflicts (HTTP 400). Defaults to 300 seconds.""" RETRY_BASE_DELAY_SECONDS = "FABRIC_CICD_RETRY_BASE_DELAY_SECONDS" """Override base delay for item name conflict retries. Defaults to 30 seconds.""" RETRY_MAX_DURATION_SECONDS = "FABRIC_CICD_RETRY_MAX_DURATION_SECONDS" """Override max duration for item name conflict retries. Defaults to 300 seconds.""" PARALLEL_MAX_WORKERS = "FABRIC_CICD_PARALLEL_MAX_WORKERS" """Override max parallel workers for concurrent item publishing. Defaults to 8.""" class ItemType(str, Enum): """Enumeration of supported Microsoft Fabric item types.""" APACHE_AIRFLOW_JOB = "ApacheAirflowJob" COPY_JOB = "CopyJob" DATA_AGENT = "DataAgent" DATA_BUILD_TOOL_JOB = "DataBuildToolJob" DATA_PIPELINE = "DataPipeline" DATAFLOW = "Dataflow" ENVIRONMENT = "Environment" EVENTHOUSE = "Eventhouse" EVENTSTREAM = "Eventstream" GRAPHQL_API = "GraphQLApi" KQL_DASHBOARD = "KQLDashboard" KQL_DATABASE = "KQLDatabase" KQL_QUERYSET = "KQLQueryset" LAKEHOUSE = "Lakehouse" MIRRORED_DATABASE = "MirroredDatabase" ML_EXPERIMENT = "MLExperiment" MOUNTED_DATA_FACTORY = "MountedDataFactory" NOTEBOOK = "Notebook" ONTOLOGY = "Ontology" REFLEX = "Reflex" REPORT = "Report" SEMANTIC_MODEL = "SemanticModel" SPARK_JOB_DEFINITION = "SparkJobDefinition" SQL_DATABASE = "SQLDatabase" USER_DATA_FUNCTION = "UserDataFunction" VARIABLE_LIBRARY = "VariableLibrary" WAREHOUSE = "Warehouse" # Serial execution order for publishing items determines dependency order. # Unpublish order is the reverse of this. SERIAL_ITEM_PUBLISH_ORDER: dict[int, ItemType] = { 1: ItemType.VARIABLE_LIBRARY, 2: ItemType.WAREHOUSE, 3: ItemType.MIRRORED_DATABASE, 4: ItemType.LAKEHOUSE, 5: ItemType.SQL_DATABASE, 6: ItemType.ENVIRONMENT, 7: ItemType.USER_DATA_FUNCTION, 8: ItemType.EVENTHOUSE, 9: ItemType.SPARK_JOB_DEFINITION, 10: ItemType.NOTEBOOK, 11: ItemType.SEMANTIC_MODEL, 12: ItemType.REPORT, 13: ItemType.COPY_JOB, 14: ItemType.DATA_BUILD_TOOL_JOB, 15: ItemType.KQL_DATABASE, 16: ItemType.KQL_QUERYSET, 17: ItemType.REFLEX, 18: ItemType.EVENTSTREAM, 19: ItemType.KQL_DASHBOARD, 20: ItemType.DATAFLOW, 21: ItemType.DATA_PIPELINE, 22: ItemType.GRAPHQL_API, 23: ItemType.APACHE_AIRFLOW_JOB, 24: ItemType.MOUNTED_DATA_FACTORY, 25: ItemType.DATA_AGENT, 26: ItemType.ML_EXPERIMENT, 27: ItemType.ONTOLOGY, } class FeatureFlag(str, Enum): """Enumeration of supported feature flags for fabric-cicd.""" ENABLE_LAKEHOUSE_UNPUBLISH = "enable_lakehouse_unpublish" """Set to enable the deletion of Lakehouses.""" ENABLE_WAREHOUSE_UNPUBLISH = "enable_warehouse_unpublish" """Set to enable the deletion of Warehouses.""" ENABLE_SQLDATABASE_UNPUBLISH = "enable_sqldatabase_unpublish" """Set to enable the deletion of SQL Databases.""" ENABLE_EVENTHOUSE_UNPUBLISH = "enable_eventhouse_unpublish" """Set to enable the deletion of Eventhouses.""" ENABLE_KQLDATABASE_UNPUBLISH = "enable_kqldatabase_unpublish" """Set to enable the deletion of KQL Databases (attached to Eventhouses).""" ENABLE_SHORTCUT_PUBLISH = "enable_shortcut_publish" """Set to enable deploying shortcuts with the lakehouse.""" DISABLE_WORKSPACE_FOLDER_PUBLISH = "disable_workspace_folder_publish" """Set to disable deploying workspace sub folders.""" CONTINUE_ON_SHORTCUT_FAILURE = "continue_on_shortcut_failure" """Set to allow deployment to continue even when shortcuts fail to publish.""" ENABLE_ENVIRONMENT_VARIABLE_REPLACEMENT = "enable_environment_variable_replacement" """Set to enable the use of pipeline variables.""" ENABLE_EXPERIMENTAL_FEATURES = "enable_experimental_features" """Set to enable experimental features, such as selective deployments.""" ENABLE_ITEMS_TO_INCLUDE = "enable_items_to_include" """Set to enable selective publishing/unpublishing of items.""" ENABLE_EXCLUDE_FOLDER = "enable_exclude_folder" """Set to enable folder-based exclusion during publish operations.""" ENABLE_INCLUDE_FOLDER = "enable_include_folder" """Set to enable folder-based inclusion during publish operations.""" ENABLE_SHORTCUT_EXCLUDE = "enable_shortcut_exclude" """Set to enable selective publishing of shortcuts in a Lakehouse.""" ENABLE_RESPONSE_COLLECTION = "enable_response_collection" """Set to enable collection of API responses during publish operations.""" ENABLE_HARD_DELETE = "enable_hard_delete" """Set to enable hard deletion of items, bypassing the workspace recycle bin.""" class OperationType(str, Enum): """Enumeration of operation types for publish/unpublish workflows.""" PUBLISH = "deployment" """Publishing items to the workspace.""" UNPUBLISH = "unpublish" """Unpublishing/removing items from the workspace.""" # The following resources can be unpublished only if their feature flags are set UNPUBLISH_FLAG_MAPPING = { ItemType.LAKEHOUSE.value: FeatureFlag.ENABLE_LAKEHOUSE_UNPUBLISH.value, ItemType.SQL_DATABASE.value: FeatureFlag.ENABLE_SQLDATABASE_UNPUBLISH.value, ItemType.WAREHOUSE.value: FeatureFlag.ENABLE_WAREHOUSE_UNPUBLISH.value, ItemType.EVENTHOUSE.value: FeatureFlag.ENABLE_EVENTHOUSE_UNPUBLISH.value, ItemType.KQL_DATABASE.value: FeatureFlag.ENABLE_KQLDATABASE_UNPUBLISH.value, } # Item Type ACCEPTED_ITEM_TYPES = tuple(item_type.value for item_type in ItemType) # API URLs DEFAULT_API_ROOT_URL = validate_env_var_api_url(EnvVar.DEFAULT_API_ROOT_URL.value, "https://api.powerbi.com") FABRIC_API_ROOT_URL = validate_env_var_api_url(EnvVar.FABRIC_API_ROOT_URL.value, "https://api.fabric.microsoft.com") # Retry Settings RETRY_AFTER_SECONDS = float(os.environ.get(EnvVar.RETRY_AFTER_SECONDS.value, 300)) RETRY_BASE_DELAY_SECONDS = float(os.environ.get(EnvVar.RETRY_BASE_DELAY_SECONDS.value, 30)) RETRY_MAX_DURATION_SECONDS = int(os.environ.get(EnvVar.RETRY_MAX_DURATION_SECONDS.value, 300)) # Parallel Settings _parallel_max_workers_raw = os.environ.get(EnvVar.PARALLEL_MAX_WORKERS.value) PARALLEL_MAX_WORKERS: int = ( int(_parallel_max_workers_raw) if _parallel_max_workers_raw and _parallel_max_workers_raw.isdigit() else 8 ) # HTTP Headers AUTHORIZATION_HEADER = "authorization" # Publish SHELL_ONLY_PUBLISH = [ ItemType.LAKEHOUSE.value, ItemType.WAREHOUSE.value, ItemType.SQL_DATABASE.value, ItemType.ML_EXPERIMENT.value, ] # Items that do not require assigned capacity NO_ASSIGNED_CAPACITY_REQUIRED = [ItemType.SEMANTIC_MODEL.value, ItemType.REPORT.value] # Exclude Path Regex Patterns for filtering files during publish EXCLUDE_PATH_REGEX_MAPPING = { ItemType.DATA_AGENT.value: r".*\.pbi[/\\].*", ItemType.REPORT.value: r".*\.pbi[/\\].*", ItemType.SEMANTIC_MODEL.value: r".*\.pbi[/\\].*", ItemType.EVENTHOUSE.value: r".*\.children[/\\].*", } # API Format Mapping for item types that require specific API formats API_FORMAT_MAPPING = { ItemType.SPARK_JOB_DEFINITION.value: "SparkJobDefinitionV2", ItemType.NOTEBOOK.value: "ipynb", } # REGEX Constants WORKSPACE_ID_REFERENCE_REGEX = r"\"?(default_lakehouse_workspace_id|workspaceId|workspace)\"?\s*[:=]\s*\"(.*?)\"" DATAFLOW_SOURCE_REGEX = ( r'(PowerPlatform\.Dataflows)(?:\(\[\]\))?[\s\S]*?workspaceId\s*=\s*"(.*?)"[\s\S]*?dataflowId\s*=\s*"(.*?)"' ) INVALID_FOLDER_CHAR_REGEX = r'[~"#.%&*:<>?/\\{|}]' KQL_DATABASE_FOLDER_PATH_REGEX = r"(?i)^(.*)/[^/]+\.Eventhouse/\.children(?:/.*)?$" # Well known file names DATA_PIPELINE_CONTENT_FILE_JSON = "pipeline-content.json" # Item Type to File Mapping (to check for item dependencies) ITEM_TYPE_TO_FILE = {ItemType.DATA_PIPELINE.value: DATA_PIPELINE_CONTENT_FILE_JSON} # Property path to get SQL Endpoint or Eventhouse URI PROPERTY_PATH_ATTR_MAPPING = { ItemType.LAKEHOUSE.value: { "sqlendpoint": "body/properties/sqlEndpointProperties/connectionString", "sqlendpointid": "body/properties/sqlEndpointProperties/id", }, ItemType.WAREHOUSE.value: { "sqlendpoint": "body/properties/connectionString", }, ItemType.SQL_DATABASE.value: { "sqlendpoint": "body/properties/serverFqdn", }, ItemType.EVENTHOUSE.value: { "queryserviceuri": "body/properties/queryServiceUri", }, } # Parameter file configs PARAMETER_FILE_NAME = "parameter.yml" # Parameters to validate PARAM_NAMES = ["find_replace", "key_value_replace", "spark_pool", "semantic_model_binding"] ITEM_ATTR_LOOKUP = ["id", "sqlendpoint", "sqlendpointid", "queryserviceuri"] # Parameter file validation messages INVALID_REPLACE_VALUE_SPARK_POOL = { "missing key": "The '{}' environment dict in spark_pool must contain a 'type' and a 'name' key", "missing value": "The '{}' environment in spark_pool is missing a value for '{}' key", "invalid value": "The '{}' environment in spark_pool must contain 'Capacity' or 'Workspace' as a value for 'type'", } PARAMETER_MSGS = { "validating": "Validating {}", "passed": "Validation passed: {}", "failed": "Validation failed with error: {}", "terminate": "Validation terminated: {}", "found": f"Found {PARAMETER_FILE_NAME} file", "not found": "Parameter file not found with path: {}", "not set": "Parameter file path is not set", "empty yaml": "YAML content is empty", "duplicate key": "duplicate key(s) found: {}", "valid load": f"Successfully loaded {PARAMETER_FILE_NAME}", "invalid load": f"Error loading {PARAMETER_FILE_NAME} " + "'{}'", "invalid structure": "Invalid parameter file structure", "valid structure": "Parameter file structure is valid", "invalid name": "Invalid parameter name '{}' found in the parameter file", "valid name": "Parameter names are valid", "invalid data type": "The provided '{}' is not of type {} in {}", "missing key": "{} is missing keys", "invalid key": "{} contains invalid keys", "valid keys": "{} contains valid keys", "mixed format": "Parameter '{}' contains mixed format keys (legacy and new format cannot be combined)", "missing required value": "Missing value for '{}' key in {}", "valid required values": "Required values in {} are valid", "missing replace value": "{} is missing a replace value for '{}' environment'", "valid replace value": "Values in 'replace_value' dict in {} are valid", "invalid replace value": INVALID_REPLACE_VALUE_SPARK_POOL, "no optional": "No optional values provided in {}", "invalid item type": "Item type '{}' not in scope", "invalid item name": "Item name '{}' not found in the repository directory", "invalid file path": "Number of paths in list '{}' that are invalid or not found in the repository directory: {}", "no valid file path": "No valid file path found in the repository directory for {}", "valid optional": "Optional values in {} are valid. Checking for file matches in the repository directory", "valid parameter": "{} parameter is valid", "skip": "The {} '{}' replacement will be skipped due to {} in parameter {}", "no target env": "target environment '{}' not found", "all target env": "The replace value: '{}' will be applied for any target environment", "other target env": "The '{}' environment key can only be used alone. Other environment keys found in replace_value: '{}'", "no filter match": "unmatched optional filters", # Path resolution messages "resolving_relative_path": "Resolving path '{}' to be relative to repository directory", "using_param_file_path": "Using parameter file path: '{}'", "using_default_param_file_path": "Using default parameter file path: '{}'", "param_file_not_found": "Parameter file path not found at: '{}'. The path was resolved from: '{}' relative to repository directory: '{}'", "param_path_not_file": "The specified parameter path '{}' exists but is not a file.", "both_param_path_and_name": "Both parameter_file_name: '{}' and parameter_file_path: '{}' were provided. Using parameter_file_path", # Parameter validation messages "param_not_found": "The {} parameter was not found", "param_found": "Found the {} parameter", "param_count": "{} {} parameters found", "regex_ignored": "The provided is_regex value is not set to 'true', regex matching will be ignored.", "validation_complete": "Parameter file validation passed", "gateway_deprecated": "The 'gateway_binding' parameter is deprecated and will be removed in future releases. Please use 'semantic_model_binding' instead.", "duplicate_semantic_model": "Duplicate semantic model names found: {}. Each semantic model should only appear once in the configuration as only one connection can be bound per semantic model. Please remove duplicate entries to avoid unpredictable binding behavior.", # Template parameter file messages "template_file_not_found": "Template parameter file not found: {}", "template_file_invalid": "Invalid template parameter file {}: {}", "template_file_error": "Error loading template parameter file {}: {}", "template_file_loaded": "Successfully loaded template parameter file: {}", "template_files_processed": "Successfully processed {} template parameter file(s)", "template_files_none_valid": "None of the template parameter files were valid or found, content will not be added", } # Wildcard path support validations WILDCARD_PATH_VALIDATIONS = [ # Invalid combinations { "check": lambda p: any(bad in p for bad in ["/**/*/", "**/**", "//", "\\\\", "**/**/"]), "message": lambda p: f"Invalid wildcard combination in pattern: '{p}'", }, # Incorrect recursive wildcard format { "check": lambda p: "**" in p and not ("**/" in p or "/**" in p), "message": lambda p: f"Invalid recursive wildcard format (use **/ or /**): '{p}'", }, ] INDENT = "->" # Define supported sections and settings for config file CONFIG_SECTIONS = { "core": { "type": dict, "settings": ["workspace_id", "workspace", "repository_directory", "item_types_in_scope", "parameter"], }, "publish": { "type": dict, "settings": [ "exclude_regex", "folder_exclude_regex", "folder_path_to_include", "items_to_include", "shortcut_exclude_regex", "skip", ], }, "unpublish": {"type": dict, "settings": ["exclude_regex", "items_to_include", "skip"]}, "features": {"type": (list, dict), "settings": []}, "constants": {"type": dict, "settings": []}, } # Config deployment validation messages CONFIG_VALIDATION_MSGS = { # File validation "file": { "path_empty": "Configuration file path must be a non-empty string", "invalid_path": "Invalid file path '{}': {}", "not_found": "Configuration file not found: {}", "not_file": "Path is not a file: {}", "yaml_syntax": "Invalid YAML syntax: {}", "encoding_error": "File encoding error (expected UTF-8): {}", "permission_denied": "Permission denied reading file: {}", "unexpected_error": "Unexpected error reading file: {}", "empty_file": "Configuration file is empty or contains only comments", "not_dict": "Configuration must be a dictionary, got {}", }, # Override validation "override": { "apply_failed": "Failed to apply config override for section '{}': {}", "unsupported_section": "Cannot override unsupported config section: '{}'. Supported: {}", "wrong_type": "Override section '{}' must be a {}, got {}", "unsupported_setting": "Cannot override unsupported setting '{}.{}'. Supported: {}", "cannot_create_core": "Cannot create 'core' section - required section must exist in the config file to override", "cannot_create_required": "Cannot create required field 'core.{}'", "cannot_create_workspace_id": "Cannot create workspace identifier 'core.{}'", }, # Structure validation "structure": { "missing_core": "Configuration must contain a 'core' section", "core_not_dict": "'core' section must be a dictionary, got {}", "missing_workspace_id": "Configuration must specify either 'workspace_id' or 'workspace' in core section", "missing_repository_dir": "Configuration must specify 'repository_directory' in core section", }, # Environment validation "environment": { "no_env_with_mappings": "Configuration contains environment mappings but no environment was provided. Please specify an environment or remove environment mappings.", "env_not_found": "Environment '{}' not found in '{}' mappings. Available: {}", "empty_mapping": "'{}' environment mapping cannot be empty", "invalid_env_key": "Environment key in '{}' must be a non-empty string, got: {}", "empty_env_value": "'{}' value for environment '{}' cannot be empty", }, # Field validation "field": { "string_or_dict": "'{}' must be either a string or environment mapping dictionary (e.g., {{dev: 'dev_value', prod: 'prod_value'}}), got type {}", "list_or_dict": "'{}' must be either a list or environment mapping dictionary (e.g., {{dev: ['dev_value1', 'dev_value2'], prod: ['prod_value']}}), got type {}", "empty_value": "'{}' cannot be empty", "empty_list": "'{}' cannot be empty if specified", "invalid_guid": "'{}' must be a valid GUID format: {}", "item_types_list_or_dict": "'item_types_in_scope' must be either a list or environment mapping dictionary (e.g., {{dev: ['Notebook'], prod: ['DataPipeline']}}), got type {}", "invalid_item_type": "Item type must be a string, got {}: {}", "unsupported_item_type_env": "Invalid item type '{}' in environment '{}'. Available types: {}", "unsupported_item_type": "Invalid item type '{}'. Available types: {}", }, # Path validation "path": { "skip": "Skipping {} path resolution due to config file validation failure", "absolute": "Using absolute {} path{}: '{}'", "git_repo": "{}{} must be in the same git repository as the configuration file. Config repository: {}, {} repository: {}", "resolved": "{} '{}' resolved relative to config path{}: '{}'", "not_found": "{} not found at resolved path{}: '{}'", "not_directory": "{} path exists but is not a directory{}: '{}'", "not_file": "{} path exists but is not a file{}: '{}'", "invalid": "Invalid {} path '{}'{}: {}", }, # Operation section validation "operation": { "unsupported_field": "'{}' field is not supported in '{}' section", "not_dict": "'{}' section must be a dictionary, got {}", "invalid_regex": "'{}' in {} is not a valid regex pattern: {}", "empty_string": "'{}' cannot be an empty string", "empty_list": "'{}' cannot be an empty list", "list_entry_type": "'{}[{}]' must be a string, got {}", "list_entry_empty": "'{}[{}]' cannot be an empty string", "features_type": "'features' section must be either a list or environment mapping dictionary, got {}", "empty_section": "'{}' section cannot be empty if specified", "empty_section_env": "'{}.{}' cannot be empty if specified", "invalid_constant_key": "Constant key in '{}' must be a non-empty string, got: {}", "unknown_constant": "Unknown constant '{}' in '{}' - this constant does not exist in fabric_cicd.constants", "folders_list_prefix": "'{}[{}]' entry must start with '/' (got '{}')", "mutually_exclusive": "Cannot specify both '{}' and '{}'. Choose one filtering strategy.", "mutually_exclusive_env": "Cannot specify both '{}' and '{}' for the same environment(s): {}. Choose one filtering strategy per environment.", }, # Log messages "log": { "override_section": "Override: {} '{}' section with value: '{}'", "override_setting": "Override: {} {}.{} with value: '{}'", "override_env_specific": "Override: updated {}.{}.{} with value: '{}'", "override_env_mapping": "Override: {}.{} added with environment mapping, with {} value: '{}'", "override_added_section": "Override: added '{}' section", }, } ================================================ FILE: src/fabric_cicd/fabric_workspace.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Module provides the FabricWorkspace class to manage and publish workspace items to the Fabric API.""" import json import logging import os import re import threading from pathlib import Path from typing import Optional import dpath from azure.core.credentials import TokenCredential from fabric_cicd import constants from fabric_cicd._common._check_utils import check_regex, check_valid_json_content, check_valid_yaml_content from fabric_cicd._common._exceptions import FailedPublishedItemStatusError, InputError, ParameterFileError, ParsingError from fabric_cicd._common._fabric_endpoint import FabricEndpoint from fabric_cicd._common._item import Item from fabric_cicd._common._logging import log_header from fabric_cicd.constants import FeatureFlag, ItemType logger = logging.getLogger(__name__) class FabricWorkspace: """A class to manage and publish workspace items to the Fabric API.""" def __init__( self, *, repository_directory: str, token_credential: TokenCredential, item_type_in_scope: Optional[list[str]] = None, environment: str = "N/A", workspace_id: Optional[str] = None, workspace_name: Optional[str] = None, **kwargs: object, ) -> None: """ Initializes the FabricWorkspace instance. Args: repository_directory: Local directory path of the repository where items are to be deployed from. token_credential: The token credential to use for API requests (e.g., AzureCliCredential, ClientSecretCredential) - required. item_type_in_scope: Item types that should be deployed for a given workspace. If omitted, defaults to all available item types. environment: The environment to be used for parameterization. workspace_id: The ID of the workspace to interact with. Either `workspace_id` or `workspace_name` must be provided. Considers only `workspace_id` if both are specified. workspace_name: The name of the workspace to interact with. Either `workspace_id` or `workspace_name` must be provided. Considers only `workspace_id` if both are specified. kwargs: Additional keyword arguments. Examples: Basic usage >>> from fabric_cicd import FabricWorkspace >>> from azure.identity import AzureCliCredential >>> workspace = FabricWorkspace( ... workspace_id="your-workspace-id", ... repository_directory="/path/to/repo", ... item_type_in_scope=["Environment", "Notebook", "DataPipeline"], ... token_credential=AzureCliCredential() # or any other TokenCredential ... ) Basic usage with workspace_name >>> from fabric_cicd import FabricWorkspace >>> from azure.identity import AzureCliCredential >>> workspace = FabricWorkspace( ... workspace_name="your-workspace-name", ... repository_directory="/path/to/repo", ... token_credential=AzureCliCredential() # or any other TokenCredential ... ) With optional parameters >>> from fabric_cicd import FabricWorkspace >>> from azure.identity import AzureCliCredential >>> workspace = FabricWorkspace( ... workspace_id="your-workspace-id", ... repository_directory="/your/path/to/repo", ... item_type_in_scope=["Environment", "Notebook", "DataPipeline"], ... environment="your-target-environment", ... token_credential=AzureCliCredential() # or any other TokenCredential ... ) With service principal credentials >>> from fabric_cicd import FabricWorkspace >>> from azure.identity import ClientSecretCredential >>> client_id = "your-client-id" >>> client_secret = "your-client-secret" >>> tenant_id = "your-tenant-id" >>> token_credential = ClientSecretCredential( ... client_id=client_id, client_secret=client_secret, tenant_id=tenant_id ... ) >>> workspace = FabricWorkspace( ... workspace_id="your-workspace-id", ... repository_directory="/your/path/to/repo", ... item_type_in_scope=["Environment", "Notebook", "DataPipeline"], ... token_credential=token_credential ... ) """ from fabric_cicd._common._validate_input import ( validate_environment, validate_item_type_in_scope, validate_repository_directory, validate_token_credential, validate_workspace_id, validate_workspace_name, ) # Validate token_credential. A TokenCredential is required to authenticate API requests token_credential = validate_token_credential(token_credential) # Initialize endpoint self.endpoint = FabricEndpoint(token_credential=token_credential) # Snapshot at construction so subsequent configure_fabric_fqdn calls for a # different workspace don't retarget this instance. self._api_root_url = constants.DEFAULT_API_ROOT_URL # Set workspace_id class variable if workspace_id: self.workspace_id = validate_workspace_id(workspace_id) elif workspace_name: self.workspace_id = self._resolve_workspace_id(validate_workspace_name(workspace_name)) else: msg = "Either workspace_name or workspace_id must be specified." raise InputError(msg, logger) # Validate and set class variables self.repository_directory: Path = validate_repository_directory(repository_directory) self.item_type_in_scope = validate_item_type_in_scope(item_type_in_scope) self.environment = validate_environment(environment) self.publish_item_name_exclude_regex = None self.publish_folder_path_exclude_regex = None self.publish_folder_path_to_include = None self.shortcut_exclude_regex = None self.items_to_include = None self.responses = None self.unpublish_responses = None self.repository_folders = {} self.repository_items = {} self.deployed_folders = {} self.deployed_items = {} # Initialize dataflow dependencies dictionary (used in dataflow item processing) self.dataflow_dependencies = {} # Initialize workspace pools cache (used in Environment item processing) self._workspace_pools_cache: Optional[list[dict]] = None self._workspace_pools_cache_lock = threading.Lock() # Initialize cache for _get_item_attribute method self._item_attribute_cache = {} self._item_attribute_cache_lock = threading.Lock() # Get parameter_file_path from kwargs self.parameter_file_path = kwargs.get("parameter_file_path") # base_api_url is no longer supported - raise error if provided if "base_api_url" in kwargs: msg = ( "Setting base_api_url is no longer supported. Please use the following instead:\n" ">>> import fabric_cicd.constants\n" ">>> constants.DEFAULT_API_ROOT_URL = ''" ) raise InputError(msg, logger) # Initialize parameter file — skipped when config-based deployment omits the # 'parameter' field, ensuring repository parameter.yml is not auto-discovered. skip_parameterization = kwargs.get("skip_parameterization", False) if not skip_parameterization: self._refresh_parameter_file() else: self.environment_parameter = {} logger.info( "Parameterization skipped: no parameter file configured/provided (environment=%s).", self.environment, ) @property def base_api_url(self) -> str: """Construct the base API URL using constants.""" return f"{self._api_root_url}/v1/workspaces/{self.workspace_id}" def _resolve_workspace_id(self, workspace_name: str) -> str: """Resolve workspace ID based on the workspace name given.""" response = self.endpoint.invoke(method="GET", url=f"{self._api_root_url}/v1/workspaces") for workspace in response["body"]["value"]: if workspace["displayName"] == workspace_name: return workspace["id"] msg = f"Workspace ID could not be resolved from workspace name: {workspace_name}." raise InputError(msg, logger) def _resolve_workspace_name(self) -> str: """Resolve workspace display name of the target workspace ID.""" response = self.endpoint.invoke(method="GET", url=f"{self._api_root_url}/v1/workspaces/{self.workspace_id}") if "displayName" in response.get("body", {}): return response["body"]["displayName"] msg = f"Workspace name could not be resolved from workspace ID: {self.workspace_id}." raise InputError(msg, logger) def _lookup_item_attribute(self, workspace_id: str, item_type: str, item_name: str, attribute_name: str) -> str: """Lookup item attribute in the specified workspace based on item type and name.""" response = self.endpoint.invoke( method="GET", url=f"{self._api_root_url}/v1/workspaces/{workspace_id}/items" ) for item in response["body"]["value"]: if item["type"] == item_type and item["displayName"] == item_name: item_guid = item["id"] if attribute_name == "id": return item_guid # For other attribute, use the item guid to get the attribute value return self._get_item_attribute(workspace_id, item_type, item_guid, item_name, attribute_name) msg = f"Failed to look up item in workspace: {workspace_id}, item_type: {item_type}, item_name: {item_name}" raise InputError(msg, logger) def _get_item_attribute( self, workspace_id: str, item_type: str, item_guid: str, item_name: str, attribute_name: str ) -> str: """Returns the attribute value of an item in the specified workspace based on item type and id""" # No need to make API calls if we don't have an item guid if not item_guid: return "" # Create a cache key for this request cache_key = (workspace_id, item_type, item_guid, item_name, attribute_name) # Check if result is already cached with self._item_attribute_cache_lock: if cache_key in self._item_attribute_cache: return self._item_attribute_cache[cache_key] # Check if this item type has property mappings if item_type not in constants.PROPERTY_PATH_ATTR_MAPPING: logger.debug(f"No property path mappings defined for {item_type}") return "" # Get the attribute mappings for this item type attribute_mappings = constants.PROPERTY_PATH_ATTR_MAPPING.get(item_type) # Check if the requested attribute is supported if attribute_name not in attribute_mappings: logger.debug( f"Attribute '{attribute_name}' not supported for {item_type} '{item_name}'. Supported: {list(attribute_mappings.keys())}" ) return "" # Get the property path for this attribute property_path = attribute_mappings[attribute_name] response = self.endpoint.invoke( method="GET", url=f"{self._api_root_url}/v1/workspaces/{workspace_id}/{item_type.lower()}s/{item_guid}", ) # Extract the attribute value using the path attribute_value = dpath.get(response, property_path, default="") if not attribute_value: msg = f"Attribute value not found for {item_type} '{item_name}'" raise InputError(msg, logger) # Cache the result before returning with self._item_attribute_cache_lock: self._item_attribute_cache[cache_key] = attribute_value return attribute_value def _get_workspace_pools(self) -> list[dict]: """Return the list of workspace custom Spark pools, fetching from the API on first call. The result is cached so that subsequent calls during the same deployment do not make additional API requests. Thread-safe via a lock. Returns: A list of pool dictionaries from the Fabric Spark custom-pools API. """ with self._workspace_pools_cache_lock: if self._workspace_pools_cache is None: # https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/list-workspace-custom-pools response = self.endpoint.invoke( method="GET", url=f"{self.base_api_url}/spark/pools", ) pools = response.get("body", {}).get("value") if isinstance(response, dict) else None if not isinstance(pools, list): msg = f"Unexpected response from Spark pools API: expected 'body.value' to be a list. Response: {response}" raise InputError(msg, logger) self._workspace_pools_cache = pools return self._workspace_pools_cache def _refresh_parameter_file(self) -> None: """Load parameters if file is present.""" from fabric_cicd._parameter._parameter import Parameter log_header(logger, "Validating Parameter File") # Initialize the parameter dict and Parameter object self.environment_parameter = {} parameter_obj = Parameter( repository_directory=self.repository_directory, item_type_in_scope=self.item_type_in_scope, environment=self.environment, parameter_file_name=constants.PARAMETER_FILE_NAME, parameter_file_path=self.parameter_file_path, ) is_valid = parameter_obj._validate_parameter_file() if is_valid: self.environment_parameter = parameter_obj.environment_parameter else: msg = "Deployment terminated due to an invalid parameter file" raise ParameterFileError(msg, logger) def _refresh_repository_items(self) -> None: """Refreshes the repository_items dictionary by scanning the repository directory.""" self.repository_items = {} empty_logical_id_paths = [] # Collect all paths with empty logical IDs visited_logical_ids = set() # Track visited logical IDs to avoid duplicates for root, _dirs, files in os.walk(self.repository_directory): directory = Path(root) # valid item directory with .platform file within if ".platform" in files: item_metadata_path = directory / ".platform" # Print a warning and skip directory if empty if not any(directory.iterdir()): logger.warning(f"Directory {directory.name} is empty.") continue # Attempt to read metadata file try: with Path.open(item_metadata_path, encoding="utf-8") as file: item_metadata = json.load(file) except FileNotFoundError as e: msg = f"{item_metadata_path} path does not exist in the specified repository. {e}" ParsingError(msg, logger) except json.JSONDecodeError as e: msg = f"Error decoding JSON in {item_metadata_path}. {e}" ParsingError(msg, logger) # Ensure required metadata fields are present if "type" not in item_metadata["metadata"] or "displayName" not in item_metadata["metadata"]: msg = f"displayName & type are required in {item_metadata_path}" raise ParsingError(msg, logger) item_type = item_metadata["metadata"]["type"] item_description = item_metadata["metadata"].get("description", "") item_name = item_metadata["metadata"]["displayName"] item_logical_id = item_metadata["config"]["logicalId"] # Check for empty logical ID and collect the path if not item_logical_id or item_logical_id.strip() == "": empty_logical_id_paths.append(str(item_metadata_path)) continue # Skip processing this item further # Validate duplicate logical IDs (skip default GUID as export API uses it as a placeholder) if item_logical_id != constants.DEFAULT_GUID: if item_logical_id in visited_logical_ids: msg = f"Duplicate logicalId '{item_logical_id}' found in {item_metadata_path}" raise FailedPublishedItemStatusError(msg, logger) visited_logical_ids.add(item_logical_id) item_path = directory relative_path = f"/{directory.relative_to(self.repository_directory).as_posix()}" # Special handling for KQLDatabase items: # .Eventhouse/.children/ directory structure, requires extracting the # parent folder path before the Eventhouse container, not just # the immediate parent directory if item_type == ItemType.KQL_DATABASE.value: pattern = re.compile(constants.KQL_DATABASE_FOLDER_PATH_REGEX) match = pattern.match(relative_path) relative_parent_path = match.group(1) if match else None else: relative_parent_path = "/".join(relative_path.split("/")[:-1]) if FeatureFlag.DISABLE_WORKSPACE_FOLDER_PUBLISH.value not in constants.FEATURE_FLAG: item_folder_id = self.repository_folders.get(relative_parent_path, "") else: item_folder_id = "" # Get the GUID if the item is already deployed item_guid = self.deployed_items.get(item_type, {}).get(item_name, Item("", "", "", "")).guid if item_type not in self.repository_items: self.repository_items[item_type] = {} # Add the item to the repository_items dictionary self.repository_items[item_type][item_name] = Item( type=item_type, name=item_name, description=item_description, guid=item_guid, logical_id=item_logical_id, path=item_path, folder_id=item_folder_id, folder_path=relative_parent_path, ) self.repository_items[item_type][item_name].collect_item_files() # If we found any empty logical IDs, raise an error with all paths if empty_logical_id_paths: if len(empty_logical_id_paths) == 1: msg = f"logicalId cannot be empty in {empty_logical_id_paths[0]}" else: paths_list = "\n - ".join(empty_logical_id_paths) msg = f"logicalId cannot be empty in the following files:\n - {paths_list}" raise ParsingError(msg, logger) def _refresh_deployed_items(self) -> None: """Refreshes the deployed_items dictionary by querying the Fabric workspace items API.""" # Get all items in workspace # https://learn.microsoft.com/en-us/rest/api/fabric/core/items/get-item response = self.endpoint.invoke(method="GET", url=f"{self.base_api_url}/items") self.deployed_items = {} self.workspace_items = {} for item in response["body"]["value"]: item_type = item["type"] item_description = item["description"] item_name = item["displayName"] item_guid = item["id"] item_folder_id = item.get("folderId", "") sql_endpoint = "" sql_endpoint_id = "" query_service_uri = "" # Add an empty dictionary if the item type hasn't been added yet if item_type not in self.deployed_items: self.deployed_items[item_type] = {} if item_type not in self.workspace_items: self.workspace_items[item_type] = {} # Get additional properties if item_type in [ItemType.LAKEHOUSE.value, ItemType.WAREHOUSE.value, ItemType.SQL_DATABASE.value]: sql_endpoint = self._get_item_attribute( self.workspace_id, item_type, item_guid, item_name, "sqlendpoint" ) sql_endpoint_id = self._get_item_attribute( self.workspace_id, item_type, item_guid, item_name, "sqlendpointid" ) if item_type in [ItemType.EVENTHOUSE.value]: query_service_uri = self._get_item_attribute( self.workspace_id, item_type, item_guid, item_name, "queryserviceuri" ) # Add item details to the deployed_items dictionary self.deployed_items[item_type][item_name] = Item( type=item_type, name=item_name, description=item_description, guid=item_guid, folder_id=item_folder_id, ) # Add item details to the workspace_items dictionary required for parameterization (public-facing attributes) self.workspace_items[item_type][item_name] = { "id": item_guid, "sqlendpoint": sql_endpoint, "sqlendpointid": sql_endpoint_id, "queryserviceuri": query_service_uri, } def _replace_logical_ids(self, raw_file: str) -> str: """ Replaces logical IDs with deployed GUIDs in the raw file content. Args: raw_file: The raw file content where logical IDs need to be replaced. """ for item_name in self.repository_items.values(): for item_details in item_name.values(): logical_id = item_details.logical_id item_guid = item_details.guid # Skip placeholder logical IDs (default GUID) used by items via export API if logical_id == constants.DEFAULT_GUID: continue if logical_id in raw_file: if item_guid == "": msg = f"Cannot replace logical ID '{logical_id}' as referenced item is not yet deployed." raise ParsingError(msg, logger) raw_file = raw_file.replace(logical_id, item_guid) return raw_file def _replace_parameters(self, file_obj: object, item_obj: object) -> str: """ Replaces values found in parameter file with the chosen environment value. Handles two parameter dictionary structures. Args: file_obj: The File object instance that provides the file content and file path. item_obj: The Item object instance that provides the item type and item name. """ from fabric_cicd._parameter._utils import ( check_replacement, extract_find_value, extract_parameter_filters, extract_replace_value, process_environment_key, replace_key_value, ) # Parse the file_obj and item_obj raw_file = file_obj.contents item_type = item_obj.type item_name = item_obj.name file_path = file_obj.file_path if "key_value_replace" in self.environment_parameter: for parameter_dict in self.environment_parameter.get("key_value_replace"): # Extract the file filter values and set the match condition input_type, input_name, input_path = extract_parameter_filters(self, parameter_dict) filter_match = check_replacement(input_type, input_name, input_path, item_type, item_name, file_path) # Perform replacement if condition is met and file contains valid JSON or YAML if filter_match: if check_valid_json_content(raw_file): raw_file = replace_key_value(self, parameter_dict, raw_file, self.environment) elif check_valid_yaml_content(raw_file): raw_file = replace_key_value(self, parameter_dict, raw_file, self.environment, is_yaml=True) if "find_replace" in self.environment_parameter: for parameter_dict in self.environment_parameter.get("find_replace"): # Extract the file filter values and set the match condition input_type, input_name, input_path = extract_parameter_filters(self, parameter_dict) filter_match = check_replacement(input_type, input_name, input_path, item_type, item_name, file_path) # Extract the find_pattern and replace_value_dict find_info = extract_find_value(parameter_dict, raw_file, filter_match) replace_value_dict = process_environment_key(self.environment, parameter_dict.get("replace_value", {})) # Replace any found references with specified environment value if conditions are met if filter_match and self.environment in replace_value_dict and find_info["has_matches"]: replace_value = extract_replace_value(self, replace_value_dict[self.environment]) if replace_value: pattern = find_info["pattern"] is_regex = find_info["is_regex"] if is_regex: # For regex patterns, use re.sub with lambda to replace only the captured group # Use string slicing to precisely replace only the captured group (group 1) # The slicing calculates relative positions: match.start(1) - match.start(0) gives # the start position of group 1 within the full match, and similarly for end position raw_file = re.sub( pattern, lambda match, repl=replace_value: ( match.group(0)[: match.start(1) - match.start(0)] + repl + match.group(0)[match.end(1) - match.start(0) :] ), raw_file, ) logger.debug( f"Replacing regex pattern '{pattern}' captured group with '{replace_value}' in {item_name}.{item_type}" ) else: # For non-regex matches, replace as before raw_file = raw_file.replace(pattern, replace_value) logger.debug(f"Replacing '{pattern}' with '{replace_value}' in {item_name}.{item_type}") return raw_file def _replace_workspace_ids(self, raw_file: str) -> str: """ Replaces feature branch workspace ID, default (i.e. 00000000-0000-0000-0000-000000000000) and non-default (actual workspace ID guid) values, with target workspace ID in the raw file content. Args: raw_file: The raw file content where workspace IDs need to be replaced. """ # Use re.sub to replace all matches return re.sub( constants.WORKSPACE_ID_REFERENCE_REGEX, lambda match: ( match.group(0).replace(constants.DEFAULT_GUID, self.workspace_id) if match.group(2) == constants.DEFAULT_GUID else match.group(0) ), raw_file, ) def _convert_id_to_name(self, item_type: str, generic_id: str, lookup_type: str) -> str: """ For a given item_type and id, returns the item name. Special handling for both deployed and repository items. Args: item_type: Type of the item (e.g., Notebook, Environment). generic_id: Logical id or item guid of the item based on lookup_type. lookup_type: Finding references in deployed file or repo file (Deployed or Repository). """ lookup_dict = self.repository_items if lookup_type == "Repository" else self.deployed_items for item_details in lookup_dict[item_type].values(): lookup_id = item_details.logical_id if lookup_type == "Repository" else item_details.guid if lookup_id == generic_id: return item_details.name # if not found return None def _convert_path_to_id(self, item_type: str, path: str) -> str: """ For a given path and item type, returns the logical id. Args: item_type: Type of the item (e.g., Notebook, Environment). path: Full path of the desired item. """ if item_type in self.repository_items: for item_details in self.repository_items[item_type].values(): if item_details.path == Path(path): return item_details.logical_id # if not found return None def _publish_item( self, item_name: str, item_type: str, exclude_path: str = r"^(?!.*)", func_process_file: Optional[callable] = None, **kwargs, ) -> None: """ Publishes or updates an item in the Fabric Workspace. Args: item_name: Name of the item to publish. item_type: Type of the item (e.g., Notebook, Environment). exclude_path: Regex string of paths to exclude. Defaults to r"^(?!.*)". func_process_file: Custom function to process file contents. Defaults to None. **kwargs: Additional keyword arguments. """ item = self.repository_items[item_type][item_name] folder_path = item.folder_path or "" # Initialize response collection for this item if responses are being tracked api_response = None # ===== FILTER ORDER (applied in _publish_item): Item Exclusion → Folder Exclusion → Folder Inclusion ===== # Note: items_to_include filtering is applied upstream in publish_all() via get_items_to_publish(). # 1. Skip publishing if the item is excluded by the regex if self.publish_item_name_exclude_regex: regex_pattern = check_regex(self.publish_item_name_exclude_regex) if regex_pattern.match(item_name): item.skip_publish = True logger.info(f"Skipping publishing of {item_type} '{item_name}' due to exclusion regex.") return # 2. Skip publishing if the item's folder path is excluded by the regex if self.publish_folder_path_exclude_regex and folder_path: regex_pattern = check_regex(self.publish_folder_path_exclude_regex) # Walk up the folder hierarchy checking each level against the exclusion regex. # Cases handled: # 1. Direct match — item's folder matches the regex (e.g., item in /A/B, regex matches /A/B) # 2. Ancestor match — item's ancestor folder matches (e.g., item in /A/B/C, regex matches /A) # 3. No match at any level — no exclusion applied, continue to next checks # Note: Root-level items (empty folder_path) are not impacted by folder path exclusion. # This ensures excluding a parent folder cascades to all descendants. path_to_check = folder_path while path_to_check: # If the current path (or ancestor) matches the exclusion pattern, skip this item if regex_pattern.search(path_to_check): item.skip_publish = True logger.info(f"Skipping publishing of {item_type} '{item_name}' due to folder path exclusion regex.") return # Move one level up by stripping the last path segment (e.g., "/a/b/c" -> "/a/b") if "/" in path_to_check and path_to_check != "": path_to_check = path_to_check.rsplit("/", 1)[0] else: # Reached the root level with no match; stop checking break # 3. Skip publishing if the item's folder path is not in the include list # If the item's folder is not in the explicit include list, skip item publish (even though folder has been created). # Note: unlike exclusion, this does NOT walk ancestors — only exact folder match is checked. # (e.g., including /A does NOT include items in /A/B, or including /A/B does NOT include items in /A, but the folder /A will still exist). if ( self.publish_folder_path_to_include and folder_path and folder_path not in self.publish_folder_path_to_include ): item.skip_publish = True logger.info( f"Skipping publishing of {item_type} '{item_name}' under {folder_path} as it is not in the include list." ) return item_guid = item.guid item_description = item.description item_files = item.item_files metadata_body = {"displayName": item_name, "type": item_type, "description": item_description} # Only shell deployment, no definition support (item_type can be overridden via kwargs) shell_only_publish = kwargs.get("shell_only_publish", item_type in constants.SHELL_ONLY_PUBLISH) if kwargs.get("creation_payload"): creation_payload = {"creationPayload": kwargs["creation_payload"]} combined_body = {**metadata_body, **creation_payload} elif shell_only_publish: combined_body = metadata_body else: item_payload = [] for file in item_files: if not re.match(exclude_path, file.relative_path): if file.type == "text" and not str(file.file_path).endswith(".platform"): # Only enable parameter replacement in Variable Library item definition files if item_type == ItemType.VARIABLE_LIBRARY.value: file.contents = self._replace_parameters(file, item) # Apply default processing for all other item definition files else: file.contents = func_process_file(self, item, file) if func_process_file else file.contents file.contents = self._replace_logical_ids(file.contents) file.contents = self._replace_parameters(file, item) file.contents = self._replace_workspace_ids(file.contents) item_payload.append(file.base64_payload) # Some item definitions require specifying the format as multiple API versions exist (i.e. Spark Job Definitions) if kwargs.get("api_format"): definition_body = {"definition": {"format": kwargs["api_format"], "parts": item_payload}} else: definition_body = {"definition": {"parts": item_payload}} combined_body = {**metadata_body, **definition_body} logger.info(f"Publishing {item_type} '{item_name}'") is_deployed = bool(item_guid) if not is_deployed: combined_body = {**combined_body, **{"folderId": item.folder_id}} # Create a new item if it does not exist # https://learn.microsoft.com/en-us/rest/api/fabric/core/items/create-item item_create_response = self.endpoint.invoke( method="POST", url=f"{self.base_api_url}/items", body=combined_body ) api_response = item_create_response item_guid = item_create_response["body"]["id"] self.repository_items[item_type][item_name].guid = item_guid elif is_deployed and not shell_only_publish: # Update the item's definition if full publish is required # https://learn.microsoft.com/en-us/rest/api/fabric/core/items/update-item-definition update_response = self.endpoint.invoke( method="POST", url=f"{self.base_api_url}/items/{item_guid}/updateDefinition?updateMetadata=True", body=definition_body, ) api_response = update_response elif is_deployed and shell_only_publish: # Remove the 'type' key as it's not supported in the update-item API metadata_body.pop("type", None) # Update the item's metadata # https://learn.microsoft.com/en-us/rest/api/fabric/core/items/update-item metadata_update_response = self.endpoint.invoke( method="PATCH", url=f"{self.base_api_url}/items/{item_guid}", body=metadata_body, ) api_response = metadata_update_response if FeatureFlag.DISABLE_WORKSPACE_FOLDER_PUBLISH.value not in constants.FEATURE_FLAG: deployed_item = self.deployed_items.get(item_type, {}).get(item_name) if is_deployed else None # Check if the folder has changed if deployed_item is not None and deployed_item.folder_id != item.folder_id: # Move the item to the correct folder if it has been moved # https://learn.microsoft.com/en-us/rest/api/fabric/core/items/move-item move_response = self.endpoint.invoke( method="POST", url=f"{self.base_api_url}/items/{item_guid}/move", body={"targetFolderId": f"{item.folder_id}"}, ) # For move operations, combine responses if we're tracking them if self.responses is not None: if api_response: # If we already have a response, combine them api_response = {"publish_response": api_response, "move_response": move_response} else: # If move is the only operation, use the move response api_response = move_response logger.debug( f"Moved {item_guid} from folder_id {self.deployed_items[item_type][item_name].folder_id} to folder_id {item.folder_id}" ) # Store response if responses are being tracked if self.responses is not None and api_response: # Initialize item_type dictionary if it doesn't exist if item_type not in self.responses: self.responses[item_type] = {} self.responses[item_type][item_name] = api_response # skip_publish_logging provided in kwargs to suppress logging if further processing is to be done if not kwargs.get("skip_publish_logging", False): logger.info(f"{constants.INDENT}Published {item_type} '{item_name}'") return def _unpublish_item(self, item_name: str, item_type: str) -> None: """ Unpublishes an item from the Fabric workspace. Args: item_name: Name of the item to unpublish. item_type: Type of the item (e.g., Notebook, Environment). """ item_guid = self.deployed_items[item_type][item_name].guid logger.info(f"Unpublishing {item_type} '{item_name}'") # Delete the item from the workspace # https://learn.microsoft.com/en-us/rest/api/fabric/core/items/delete-item try: # Apply hard delete if the feature flag is enabled, otherwise defaults to soft deleting (moves the item to the recycle bin) hard_delete = FeatureFlag.ENABLE_HARD_DELETE.value in constants.FEATURE_FLAG delete_url = f"{self.base_api_url}/items/{item_guid}" + ("?hardDelete=true" if hard_delete else "") api_response = self.endpoint.invoke(method="DELETE", url=delete_url) logger.info(f"{constants.INDENT}Unpublished {item_type} '{item_name}'") # Store response if responses are being tracked if self.unpublish_responses is not None and api_response: self.unpublish_responses.setdefault(item_type, {})[item_name] = api_response except Exception as e: msg = f"Failed to unpublish {item_type} '{item_name}'. Raw exception: {e}" if not hard_delete: msg += ( f" Consider enabling the '{FeatureFlag.ENABLE_HARD_DELETE.value}' feature flag" " to perform a permanent deletion, which bypasses the recycle bin" " and may resolve this issue (requires workspace Admin role)." ) logger.warning(msg) def _refresh_deployed_folders(self) -> None: """ Converts the folder list payload into a structure of folder name and their ids output should be like this: { "/Pipeline": "323eaa75-d70b-498c-8544-6c4219bf336e", "/Notebook": "f802fd90-c70e-4d77-b079-538f617646d3", "/Notebook/Processing": "36ed1a63-be82-4a7a-9364-2e4ff3a66b31" } """ self.deployed_folders = {} request_url = f"{self.base_api_url}/folders" folders = [] while request_url: # https://learn.microsoft.com/en-us/rest/api/fabric/core/folders/list-folders response = self.endpoint.invoke(method="GET", url=request_url) # Handle cases where the response body is empty folder_response = response["body"].get("value", []) folders.extend(folder for folder in folder_response) request_url = response["header"].get("continuationUri", None) # Create a lookup table for folders by their ID folder_lookup = {folder["id"]: folder for folder in folders} # Build the folder hierarchy folder_hierarchy = {} def get_full_path(folder: dict) -> str: """Recursively build the full path for a folder""" parent_id = folder.get("parentFolderId") if parent_id: parent_folder = folder_lookup.get(parent_id) if parent_folder: return f"{get_full_path(parent_folder)}/{folder['displayName']}" return f"/{folder['displayName']}" for folder in folders: full_path = get_full_path(folder) folder_hierarchy[full_path] = folder["id"] self.deployed_folders = folder_hierarchy def _refresh_repository_folders(self) -> None: """ Converts the folder list payload into a structure of folder name and their ids, skipping empty folders or folders that only contain other empty folders. output should be like this: { "/Pipeline": "", "/Notebook": "", "/Notebook/Processing": "" } """ self.repository_folders = {} root_path = Path(self.repository_directory) folder_hierarchy = {} # Collect all folders that directly contain a .platform file platform_folders = set(p.parent for p in root_path.rglob(".platform")) # Now, for every folder, check if any of its subfolders is in platform_folders for folder in root_path.rglob("*"): if not folder.is_dir() or folder == root_path or folder.name == ".children": continue # Skip folders that directly contain a .platform file if folder in platform_folders: continue # Check if any subfolder (at any depth) is in platform_folders if any(sub in platform_folders for sub in folder.rglob("*") if sub.is_dir()): relative_path = f"/{folder.relative_to(root_path).as_posix()}" folder_hierarchy[relative_path] = "" self.repository_folders = folder_hierarchy def _publish_folders(self) -> None: """Publishes all folders from the repository.""" # Sort folders by the number of '/' in their paths (ascending order) sorted_folders = sorted(self.repository_folders.keys(), key=lambda path: path.count("/")) log_header(logger, "Publishing Workspace Folders") logger.info("Publishing Workspace Folders") for folder_path in sorted_folders: # Skip folders matching the exclusion regex if self.publish_folder_path_exclude_regex: regex_pattern = check_regex(self.publish_folder_path_exclude_regex) if regex_pattern.search(folder_path): logger.info(f"Skipping publishing of folder '{folder_path}' due to folder path exclusion regex.") continue # If any ancestor folder was excluded by the regex, skip this # descendant folder too to preserve a consistent hierarchy ancestor_path = folder_path ancestor_excluded = False while "/" in ancestor_path and ancestor_path != "": ancestor_path = ancestor_path.rsplit("/", 1)[0] if ancestor_path and regex_pattern.search(ancestor_path): ancestor_excluded = True break if ancestor_excluded: logger.info( f"Skipping publishing of folder '{folder_path}' as its ancestor folder was excluded by regex." ) continue # Skip folders not in the include list # Ancestor folders must be published to preserve the correct hierarchy. # Even though they may not be explicitly included, (e.g., if /A/B is included, /A must also be published). if self.publish_folder_path_to_include: is_included = folder_path in self.publish_folder_path_to_include is_ancestor_of_included = any( included.startswith(folder_path + "/") for included in self.publish_folder_path_to_include ) if not is_included and not is_ancestor_of_included: logger.info(f"Skipping publishing of folder '{folder_path}' as it is not in the include list.") continue if folder_path in self.deployed_folders: # Folder already deployed, update local hierarchy self.repository_folders[folder_path] = self.deployed_folders[folder_path] logger.debug(f"Folder exists: {folder_path}") continue # Publish the folder folder_name = folder_path.split("/")[-1] folder_parent_path = "/".join(folder_path.split("/")[:-1]) folder_parent_id = self.repository_folders.get(folder_parent_path, None) if re.search(constants.INVALID_FOLDER_CHAR_REGEX, folder_name): msg = f"Folder name '{folder_name}' contains invalid characters." raise InputError(msg, logger) request_body = {"displayName": folder_name} if folder_parent_id: request_body["parentFolderId"] = folder_parent_id request_url = f"{self.base_api_url}/folders" response = self.endpoint.invoke(method="POST", url=request_url, body=request_body) # Update local hierarchy with the new folder ID self.repository_folders[folder_path] = response["body"]["id"] logger.debug(f"Published folder: {folder_path}") logger.info(f"{constants.INDENT}Published") def _unpublish_folders(self) -> None: """Unpublishes all empty folders in workspace.""" # Sort folders by the number of '/' in their paths (descending order) sorted_folder_ids = [ self.deployed_folders[key] for key in sorted(self.deployed_folders.keys(), key=lambda path: path.count("/"), reverse=True) ] ## Any folder that neither contains items nor is an ancestor of a folder ## containing items is considered orphaned # Create a set of folders that contain items unorphaned_folders = { item.folder_id for items in self.deployed_items.values() for item in items.values() if item.folder_id } # Skip deletion if all deployed folders are unorphaned if unorphaned_folders == set(sorted_folder_ids): return # Create a reversed mapping for folder_id to folder_path lookups folder_id_to_path_mapping = {folder_id: folder_path for folder_path, folder_id in self.deployed_folders.items()} # Create a copy of the unorphaned_folders set to safely iterate while modifying the original set folder_lookup = unorphaned_folders.copy() # For each folder containing items, identify and protect all its ancestor folders from deletion for folder_id in folder_lookup: if folder_id in folder_id_to_path_mapping: # Get the folder path folder_path = folder_id_to_path_mapping[folder_id] # Move up the folder hierarchy and add all ancestor folders current_folder_path = folder_path while current_folder_path != "/": # Get the parent folder path current_folder_path = current_folder_path.rsplit("/", 1)[0] or "/" # Get the folder_id for this path and add to the unorphaned_folder set parent_folder_id = self.deployed_folders.get(current_folder_path) if parent_folder_id: unorphaned_folders.add(parent_folder_id) # Check if deletion can be skipped after update to unorphaned_folder set if unorphaned_folders == set(sorted_folder_ids): return logger.info("Unpublishing Workspace Folders") # Pop all folders for folder_id in sorted_folder_ids: if folder_id not in unorphaned_folders: # Folder deployed, but not in repository # Delete the folder from the workspace # https://learn.microsoft.com/en-us/rest/api/fabric/core/folders/delete-folder try: self.endpoint.invoke(method="DELETE", url=f"{self.base_api_url}/folders/{folder_id}") logger.debug(f"Unpublished folder: {folder_id}") except Exception as e: logger.warning(f"Failed to unpublish folder {folder_id}. Raw exception: {e}") logger.info(f"{constants.INDENT}Unpublished") ================================================ FILE: src/fabric_cicd/publish.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Module for publishing and unpublishing Fabric workspace items.""" import logging from typing import Optional import dpath from azure.core.credentials import TokenCredential import fabric_cicd._items as items from fabric_cicd import constants from fabric_cicd._common._config_utils import ( config_overrides_scope, extract_publish_settings, extract_unpublish_settings, extract_workspace_settings, load_config_file, ) from fabric_cicd._common._deployment_result import DeploymentResult, DeploymentStatus from fabric_cicd._common._exceptions import FailedPublishedItemStatusError, InputError from fabric_cicd._common._logging import log_header from fabric_cicd._common._validate_input import ( validate_environment, validate_fabric_workspace_obj, validate_folder_path_exclude_regex, validate_folder_path_to_include, validate_items_to_include, validate_shortcut_exclude_regex, ) from fabric_cicd.constants import FeatureFlag, ItemType from fabric_cicd.fabric_workspace import FabricWorkspace logger = logging.getLogger(__name__) def publish_all_items( fabric_workspace_obj: FabricWorkspace, item_name_exclude_regex: Optional[str] = None, folder_path_exclude_regex: Optional[str] = None, folder_path_to_include: Optional[list[str]] = None, items_to_include: Optional[list[str]] = None, shortcut_exclude_regex: Optional[str] = None, ) -> Optional[dict]: """ Publishes all items defined in the `item_type_in_scope` list of the given FabricWorkspace object. Args: fabric_workspace_obj: The FabricWorkspace object containing the items to be published. item_name_exclude_regex: Regex pattern to exclude specific items from being published. folder_path_exclude_regex: Regex pattern matched against folder paths (e.g., "/folder_name") to exclude folders and their items from being published. folder_path_to_include: List of folder paths in the format "/folder_name"; only the specified folders and their items will be published. items_to_include: List of items in the format "item_name.item_type" that should be published. shortcut_exclude_regex: Regex pattern to exclude specific shortcuts from being published in lakehouses. Returns: Dict containing all API responses if the ``enable_response_collection`` feature flag is enabled and at least one response was collected; otherwise, None. folder_path_exclude_regex: This is an experimental feature in fabric-cicd. Use at your own risk as selective deployments are not recommended due to item dependencies. Cannot be used together with ``folder_path_to_include`` for the same environment. To enable this feature, see How To -> Optional Features for information on which flags to enable. folder_path_to_include: This is an experimental feature in fabric-cicd. Use at your own risk as selective deployments are not recommended due to item dependencies. Cannot be used together with ``folder_path_exclude_regex`` for the same environment. To enable this feature, see How To -> Optional Features for information on which flags to enable. items_to_include: This is an experimental feature in fabric-cicd. Use at your own risk as selective deployments are not recommended due to item dependencies. To enable this feature, see How To -> Optional Features for information on which flags to enable. shortcut_exclude_regex: This is an experimental feature in fabric-cicd. Use at your own risk as selective shortcut deployments may result in missing data dependencies. To enable this feature, see How To -> Optional Features for information on which flags to enable. Examples: Basic usage >>> from fabric_cicd import FabricWorkspace, publish_all_items >>> from azure.identity import AzureCliCredential >>> workspace = FabricWorkspace( ... workspace_id="your-workspace-id", ... repository_directory="/path/to/repo", ... item_type_in_scope=["Environment", "Notebook", "DataPipeline"], ... token_credential=AzureCliCredential() # or any other TokenCredential ... ) >>> publish_all_items(workspace) With regex name exclusion >>> from fabric_cicd import FabricWorkspace, publish_all_items >>> from azure.identity import AzureCliCredential >>> workspace = FabricWorkspace( ... workspace_id="your-workspace-id", ... repository_directory="/path/to/repo", ... item_type_in_scope=["Environment", "Notebook", "DataPipeline"], ... token_credential=AzureCliCredential() # or any other TokenCredential ... ) >>> exclude_regex = ".*_do_not_publish" >>> publish_all_items(workspace, item_name_exclude_regex=exclude_regex) With folder exclusion >>> from fabric_cicd import FabricWorkspace, publish_all_items, append_feature_flag >>> from azure.identity import AzureCliCredential >>> append_feature_flag("enable_experimental_features") >>> append_feature_flag("enable_exclude_folder") >>> workspace = FabricWorkspace( ... workspace_id="your-workspace-id", ... repository_directory="/path/to/repo", ... item_type_in_scope=["Environment", "Notebook", "DataPipeline"], ... token_credential=AzureCliCredential() # or any other TokenCredential ... ) >>> folder_exclude_regex = "^/legacy" >>> publish_all_items(workspace, folder_path_exclude_regex=folder_exclude_regex) With folder inclusion >>> from fabric_cicd import FabricWorkspace, publish_all_items, append_feature_flag >>> from azure.identity import AzureCliCredential >>> append_feature_flag("enable_experimental_features") >>> append_feature_flag("enable_include_folder") >>> workspace = FabricWorkspace( ... workspace_id="your-workspace-id", ... repository_directory="/path/to/repo", ... item_type_in_scope=["Environment", "Notebook", "DataPipeline"], ... token_credential=AzureCliCredential() # or any other TokenCredential ... ) >>> folder_path_to_include = ["/subfolder"] >>> publish_all_items(workspace, folder_path_to_include=folder_path_to_include) With items to include >>> from fabric_cicd import FabricWorkspace, publish_all_items, append_feature_flag >>> from azure.identity import AzureCliCredential >>> append_feature_flag("enable_experimental_features") >>> append_feature_flag("enable_items_to_include") >>> workspace = FabricWorkspace( ... workspace_id="your-workspace-id", ... repository_directory="/path/to/repo", ... item_type_in_scope=["Environment", "Notebook", "DataPipeline"], ... token_credential=AzureCliCredential() # or any other TokenCredential ... ) >>> items_to_include = ["Hello World.Notebook", "Hello.Environment"] >>> publish_all_items(workspace, items_to_include=items_to_include) With shortcut exclusion >>> from fabric_cicd import FabricWorkspace, publish_all_items, append_feature_flag >>> from azure.identity import AzureCliCredential >>> append_feature_flag("enable_experimental_features") >>> append_feature_flag("enable_shortcut_exclude") >>> append_feature_flag("enable_shortcut_publish") >>> workspace = FabricWorkspace( ... workspace_id="your-workspace-id", ... repository_directory="/path/to/repo", ... item_type_in_scope=["Lakehouse"], ... token_credential=AzureCliCredential() # or any other TokenCredential ... ) >>> shortcut_exclude_regex = "^temp_.*" # Exclude shortcuts starting with "temp_" >>> publish_all_items(workspace, shortcut_exclude_regex=shortcut_exclude_regex) With response collection >>> from fabric_cicd import FabricWorkspace, publish_all_items, append_feature_flag >>> from azure.identity import AzureCliCredential >>> append_feature_flag("enable_response_collection") >>> workspace = FabricWorkspace( ... workspace_id="your-workspace-id", ... repository_directory="/path/to/repo", ... item_type_in_scope=["Environment", "Notebook", "DataPipeline"], ... token_credential=AzureCliCredential() # or any other TokenCredential ... ) >>> responses = publish_all_items(workspace) >>> # Access all responses >>> print(responses) >>> # Access individual item response (dict with "header", "body", "status_code" keys) >>> notebook_response = workspace.responses["Notebook"]["Hello World"] >>> print(notebook_response["status_code"]) # e.g., 200 With get_changed_items (deploy only git-changed items) >>> from fabric_cicd import FabricWorkspace, publish_all_items, get_changed_items >>> from azure.identity import AzureCliCredential >>> workspace = FabricWorkspace( ... workspace_id="your-workspace-id", ... repository_directory="/path/to/repo", ... item_type_in_scope=["Notebook", "DataPipeline"], ... token_credential=AzureCliCredential() # or any other TokenCredential ... ) >>> changed = get_changed_items(workspace.repository_directory) >>> if changed: ... publish_all_items(workspace, items_to_include=changed) """ fabric_workspace_obj = validate_fabric_workspace_obj(fabric_workspace_obj) responses_enabled = FeatureFlag.ENABLE_RESPONSE_COLLECTION.value in constants.FEATURE_FLAG # Initialize response collection if feature flag is enabled if responses_enabled: fabric_workspace_obj.responses = {} # Check if workspace has assigned capacity, if not, exit has_assigned_capacity = None response_state = fabric_workspace_obj.endpoint.invoke( method="GET", url=f"{constants.DEFAULT_API_ROOT_URL}/v1/workspaces/{fabric_workspace_obj.workspace_id}" ) has_assigned_capacity = dpath.get(response_state, "body/capacityId", default=None) if not has_assigned_capacity and not set(fabric_workspace_obj.item_type_in_scope).issubset( set(constants.NO_ASSIGNED_CAPACITY_REQUIRED) ): msg = f"Workspace {fabric_workspace_obj.workspace_id} does not have an assigned capacity. Please assign a capacity before publishing items." raise FailedPublishedItemStatusError(msg, logger) if FeatureFlag.DISABLE_WORKSPACE_FOLDER_PUBLISH.value not in constants.FEATURE_FLAG: if folder_path_exclude_regex is not None and folder_path_to_include is not None: msg = "Cannot use both 'folder_path_exclude_regex' and 'folder_path_to_include' simultaneously. Choose one filtering strategy." raise InputError(msg, logger) if folder_path_exclude_regex is not None: validate_folder_path_exclude_regex(folder_path_exclude_regex) fabric_workspace_obj.publish_folder_path_exclude_regex = folder_path_exclude_regex if folder_path_to_include is not None: validate_folder_path_to_include(folder_path_to_include) fabric_workspace_obj.publish_folder_path_to_include = folder_path_to_include fabric_workspace_obj._refresh_deployed_folders() fabric_workspace_obj._refresh_repository_folders() fabric_workspace_obj._publish_folders() fabric_workspace_obj._refresh_deployed_items() fabric_workspace_obj._refresh_repository_items() if item_name_exclude_regex: logger.warning( "Using item_name_exclude_regex is risky as it can prevent needed dependencies from being deployed. Use at your own risk." ) fabric_workspace_obj.publish_item_name_exclude_regex = item_name_exclude_regex if items_to_include: validate_items_to_include(items_to_include, operation=constants.OperationType.PUBLISH) fabric_workspace_obj.items_to_include = items_to_include if shortcut_exclude_regex: validate_shortcut_exclude_regex(shortcut_exclude_regex) fabric_workspace_obj.shortcut_exclude_regex = shortcut_exclude_regex # Publish items in the defined order synchronously total_item_types = len(constants.SERIAL_ITEM_PUBLISH_ORDER) publishers_with_async_check: list[items.ItemPublisher] = [] for order_num, item_type in items.ItemPublisher.get_item_types_to_publish(fabric_workspace_obj): log_header(logger, f"Publishing Item {order_num}/{total_item_types}: {item_type.value}") publisher = items.ItemPublisher.create(item_type, fabric_workspace_obj) publisher.publish_all() if publisher.has_async_publish_check: publishers_with_async_check.append(publisher) # Check asynchronous publish status for relevant item types for publisher in publishers_with_async_check: log_header(logger, f"Checking {publisher.item_type} Publish State") publisher.post_publish_all_check() # Return response data if feature flag is enabled and responses were collected return fabric_workspace_obj.responses if responses_enabled and fabric_workspace_obj.responses else None def unpublish_all_orphan_items( fabric_workspace_obj: FabricWorkspace, item_name_exclude_regex: str = "^$", items_to_include: Optional[list[str]] = None, ) -> Optional[dict]: """ Unpublishes all orphaned items not present in the repository except for those matching the exclude regex. Args: fabric_workspace_obj: The FabricWorkspace object containing the items to be unpublished. item_name_exclude_regex: Regex pattern to exclude specific items from being unpublished. Default is '^$' which will exclude nothing. items_to_include: List of items in the format "item_name.item_type" that should be unpublished. Returns: Dict containing all collected API responses if the ``enable_response_collection`` feature flag is enabled and at least one response was collected; otherwise, None. Note: By default, the Fabric Delete Item API moves deleted items to the workspace recycle bin. However, not all item types support soft delete; for those types, deletion requires the ``enable_hard_delete`` feature flag. Enabling this flag bypasses the recycle bin and permanently deletes items. Hard delete requires the workspace **Admin** role. items_to_include: This is an experimental feature in fabric-cicd. Use at your own risk as selective unpublishing is not recommended due to item dependencies. To enable this feature, see How To -> Optional Features for information on which flags to enable. Examples: Basic usage >>> from fabric_cicd import FabricWorkspace, publish_all_items, unpublish_all_orphan_items >>> from azure.identity import AzureCliCredential >>> workspace = FabricWorkspace( ... workspace_id="your-workspace-id", ... repository_directory="/path/to/repo", ... item_type_in_scope=["Environment", "Notebook", "DataPipeline"], ... token_credential=AzureCliCredential() # or any other TokenCredential ... ) >>> publish_all_items(workspace) >>> unpublish_all_orphan_items(workspace) With regex name exclusion >>> from fabric_cicd import FabricWorkspace, publish_all_items, unpublish_all_orphan_items >>> from azure.identity import AzureCliCredential >>> workspace = FabricWorkspace( ... workspace_id="your-workspace-id", ... repository_directory="/path/to/repo", ... item_type_in_scope=["Environment", "Notebook", "DataPipeline"], ... token_credential=AzureCliCredential() ... ) >>> publish_all_items(workspace) >>> exclude_regex = ".*_do_not_delete" >>> unpublish_all_orphan_items(workspace, item_name_exclude_regex=exclude_regex) With items to include >>> from fabric_cicd import FabricWorkspace, publish_all_items, unpublish_all_orphan_items, append_feature_flag >>> from azure.identity import AzureCliCredential >>> append_feature_flag("enable_experimental_features") >>> append_feature_flag("enable_items_to_include") >>> workspace = FabricWorkspace( ... workspace_id="your-workspace-id", ... repository_directory="/path/to/repo", ... item_type_in_scope=["Environment", "Notebook", "DataPipeline"], ... token_credential=AzureCliCredential() # or any other TokenCredential ... ) >>> publish_all_items(workspace) >>> items_to_include = ["Hello World.Notebook", "Run Hello World.DataPipeline"] >>> unpublish_all_orphan_items(workspace, items_to_include=items_to_include) With response collection >>> from fabric_cicd import FabricWorkspace, publish_all_items, unpublish_all_orphan_items, append_feature_flag >>> from azure.identity import AzureCliCredential >>> append_feature_flag("enable_response_collection") >>> workspace = FabricWorkspace( ... workspace_id="your-workspace-id", ... repository_directory="/path/to/repo", ... item_type_in_scope=["Environment", "Notebook", "DataPipeline"], ... token_credential=AzureCliCredential() # or any other TokenCredential ... ) >>> publish_all_items(workspace) >>> responses = unpublish_all_orphan_items(workspace) >>> # Access all unpublish responses >>> print(responses) >>> # Access individual item response (dict with "header", "body", "status_code" keys) >>> notebook_response = workspace.unpublish_responses["Notebook"]["Hello World"] >>> print(notebook_response["status_code"]) # e.g., 200 """ fabric_workspace_obj = validate_fabric_workspace_obj(fabric_workspace_obj) validate_items_to_include(items_to_include, operation=constants.OperationType.UNPUBLISH) responses_enabled = FeatureFlag.ENABLE_RESPONSE_COLLECTION.value in constants.FEATURE_FLAG # Initialize response collection if feature flag is enabled if responses_enabled: fabric_workspace_obj.unpublish_responses = {} fabric_workspace_obj._refresh_deployed_items() fabric_workspace_obj._refresh_repository_items() log_header(logger, "Unpublishing Orphaned Items") # Build unpublish order based on reversed publish order, scope, and feature flags for item_type in items.ItemPublisher.get_item_types_to_unpublish(fabric_workspace_obj): to_delete_list = items.ItemPublisher.get_orphaned_items( fabric_workspace_obj, item_type, item_name_exclude_regex=item_name_exclude_regex if not items_to_include else None, items_to_include=items_to_include, ) if items_to_include and to_delete_list: logger.debug(f"Items to include for unpublishing ({item_type}): {to_delete_list}") publisher = items.ItemPublisher.create(ItemType(item_type), fabric_workspace_obj) if to_delete_list and publisher.has_dependency_tracking: to_delete_list = publisher.get_unpublish_order(to_delete_list) for item_name in to_delete_list: fabric_workspace_obj._unpublish_item(item_name=item_name, item_type=item_type) fabric_workspace_obj._refresh_deployed_items() fabric_workspace_obj._refresh_deployed_folders() if FeatureFlag.DISABLE_WORKSPACE_FOLDER_PUBLISH.value not in constants.FEATURE_FLAG: fabric_workspace_obj._unpublish_folders() # Return response data if feature flag is enabled and responses were collected return ( fabric_workspace_obj.unpublish_responses if responses_enabled and fabric_workspace_obj.unpublish_responses else None ) def deploy_with_config( config_file_path: str, *, token_credential: TokenCredential, environment: str = "N/A", config_override: Optional[dict] = None, ) -> DeploymentResult: """ Deploy items using YAML configuration file with environment-specific settings. This function provides a simplified deployment interface that loads configuration from a YAML file and executes deployment operations based on environment-specific settings. It constructs the necessary FabricWorkspace object internally and handles publish/unpublish operations according to the configuration. Args: config_file_path: Path to the YAML configuration file as a string. token_credential: Azure token credential for authentication (e.g., AzureCliCredential, ClientSecretCredential) - required. environment: Environment name to use for deployment (e.g., 'dev', 'test', 'prod'), if missing defaults to 'N/A'. config_override: Optional dictionary to override specific configuration values. Returns: DeploymentResult: A result object containing the deployment status, message, and responses (opt-in). The status will be DeploymentStatus.COMPLETED on success. The responses field contains a dictionary with ``"publish"`` and/or ``"unpublish"`` keys mapping to their respective API response data when the ``enable_response_collection`` feature flag is enabled and responses were collected, otherwise None. Raises: InputError: If configuration is invalid, environment not found, or input validation fails. ConfigValidationError: If configuration file is missing or fails structural validation. Note: On failure, the raised exception will have a ``deployment_result`` attribute containing a ``DeploymentResult`` with ``status`` set to ``DeploymentStatus.FAILED``, ``message`` set to the error description, and ``responses`` containing any partial API responses collected before the failure (requires the ``enable_response_collection`` feature flag, otherwise None). Examples: Basic usage >>> from fabric_cicd import deploy_with_config >>> from azure.identity import AzureCliCredential >>> credential = AzureCliCredential() >>> result = deploy_with_config( ... config_file_path="workspace/config.yml", ... token_credential=credential, ... environment="prod" ... ) >>> print(result.status) # DeploymentStatus.COMPLETED >>> print(result.message) # "Deployment completed successfully" >>> print(result.responses) # {"publish": {...}, "unpublish": {...}} or None With custom authentication >>> from fabric_cicd import deploy_with_config >>> from azure.identity import ClientSecretCredential >>> credential = ClientSecretCredential(tenant_id, client_id, client_secret) >>> result = deploy_with_config( ... config_file_path="workspace/config.yml", ... token_credential=credential, ... environment="prod" ... ) With override configuration >>> from fabric_cicd import deploy_with_config >>> from azure.identity import ClientSecretCredential >>> credential = ClientSecretCredential(tenant_id, client_id, client_secret) >>> result = deploy_with_config( ... config_file_path="workspace/config.yml", ... token_credential=credential, ... environment="prod", ... config_override={ ... "core": { ... "item_types_in_scope": ["Notebook"] ... }, ... "publish": { ... "skip": { ... "prod": False ... } ... } ... } ... ) Handling deployment failures >>> from fabric_cicd import deploy_with_config >>> from azure.identity import AzureCliCredential >>> credential = AzureCliCredential() >>> try: ... result = deploy_with_config( ... config_file_path="workspace/config.yml", ... token_credential=credential, ... environment="prod" ... ) ... print(result.status) # DeploymentStatus.COMPLETED ... print(result.message) # "Deployment completed successfully" ... print(result.responses) # {"publish": {...}, "unpublish": {...}} or None ... except Exception as e: ... print(e.deployment_result.status) # DeploymentStatus.FAILED ... print(e.deployment_result.message) # Original error message ... print(e.deployment_result.responses) # Partial API responses or None """ log_header(logger, "Config-Based Deployment") logger.info(f"Loading configuration from {config_file_path} for environment '{environment}'") # Initialize workspace as None so it exists in except block scope workspace = None responses_enabled = False try: # Validate environment environment = validate_environment(environment) # Load and validate configuration file config = load_config_file(config_file_path, environment, config_override) # Extract environment-specific settings workspace_settings = extract_workspace_settings(config, environment) publish_settings = extract_publish_settings(config, environment) unpublish_settings = extract_unpublish_settings(config, environment) # Apply feature flags and constants if specified with config_overrides_scope(config, environment): # Determine if response collection flag has been enabled in the config file responses_enabled = FeatureFlag.ENABLE_RESPONSE_COLLECTION.value in constants.FEATURE_FLAG # When no parameter file is configured or resolved for this environment, # parameter_file_path is None and parameterization must be skipped entirely — # any parameter.yml that happens to exist in the repository must NOT be auto-discovered. skip_parameterization = workspace_settings.get("parameter_file_path") is None # Create FabricWorkspace object with extracted settings workspace = FabricWorkspace( repository_directory=workspace_settings["repository_directory"], item_type_in_scope=workspace_settings.get("item_types_in_scope"), environment=environment, workspace_id=workspace_settings.get("workspace_id"), workspace_name=workspace_settings.get("workspace_name"), token_credential=token_credential, parameter_file_path=workspace_settings.get("parameter_file_path"), skip_parameterization=skip_parameterization, ) # Execute deployment operations based on skip settings if not publish_settings.get("skip", False): publish_all_items( workspace, item_name_exclude_regex=publish_settings.get("exclude_regex"), folder_path_exclude_regex=publish_settings.get("folder_exclude_regex"), folder_path_to_include=publish_settings.get("folder_path_to_include"), items_to_include=publish_settings.get("items_to_include"), shortcut_exclude_regex=publish_settings.get("shortcut_exclude_regex"), ) else: logger.info(f"Skipping publish operation for environment '{environment}'") if not unpublish_settings.get("skip", False): unpublish_all_orphan_items( workspace, item_name_exclude_regex=unpublish_settings.get("exclude_regex", "^$"), items_to_include=unpublish_settings.get("items_to_include"), ) else: logger.info(f"Skipping unpublish operation for environment '{environment}'") except Exception as e: e.deployment_result = DeploymentResult( status=DeploymentStatus.FAILED, message=str(e), responses=_collect_responses(workspace, responses_enabled), ) raise logger.info("Config-based deployment completed successfully") return DeploymentResult( status=DeploymentStatus.COMPLETED, message="Deployment completed successfully", responses=_collect_responses(workspace, responses_enabled), ) def _collect_responses(workspace: Optional[FabricWorkspace], responses_enabled: bool) -> Optional[dict]: """Return collected API responses if available, otherwise None.""" if not responses_enabled or workspace is None: return None result = {} if workspace.responses: result["publish"] = workspace.responses if workspace.unpublish_responses: result["unpublish"] = workspace.unpublish_responses return result or None ================================================ FILE: tests/fixtures/.gitignore ================================================ # Only the gzipped http trace should be committed to git # Snapshots are not human reviewable in a PR, it's too large. http_trace.json ================================================ FILE: tests/fixtures/README.md ================================================ # Test Fixtures ## `mock_fabric_server`: A mock Fabric REST API ### Use Cases Basically, this fixture allows Integration Testing without the costs associated with E2E tests. If you peek inside `http_trace.json`, each full deployment of `fabric-cicd` project makes thousands of API calls. As the project grows in scope, so does the amount of tests that need to be written to exercise coverage. This becomes unruly via PyTest mocks or [monkey patching](https://docs.pytest.org/en/stable/how-to/monkeypatch.html). In an ideal scenario, whenver a PR is opened, we'd test it against a real Fabric Workspace, but - stateful tests are difficult, any outages in Fabric API can cause PRs to fail - etc; this means testing against a real workspace is not realistic. But what if we could _mimic_ the Fabric API locally? That's what this fixture provides. This is a mock REST API Server that can mimic any REST API, including `https://api.powerbi.com` and `https://api.fabric.microsoft.com`. We can add any number of further mocks in the future as well - as long as the API calls are traced and snapshotted. The idea is, to exercise the public facing `fabric_cicd` API E2E rapidly. The mock server loads an `http_trace.json` file to dictate the behavior. ### Why not VCR This is very similar to [VCR Cassettes](https://github.com/vcr/vcr). The downside of VCR is, the actual HTTP interactions are abstracted away via the SDK, and as a result, the test code does not truly make calls to a real REST API that we have full control over. For example, in our mock REST API, we can add logic to override exceptions at a route level, which is not possible in VCR (without opening a PR). > In both VCR and this fixture, the one thing is common, generating snapshots takes time, since you have to interact > with the real control plane (in this case Fabric), which is costly. > > The way to think about the situation is, we're able to enjoy near 100% test coverage for _all_ of our customer facing calls > without using a real control plane, this allows us to _significantly_ increase code velocity as all code paths are tested by > nature of a full deployment. ### What is this? The 4 steps outlined in the image below are as follows: 1. Add new workloads into the codebase 2. Capture REST calls from Fabric using `debug_trace_deployment.py` 3. Move `http_trace.json.gz` into fixture 4. Enjoy rapid test coverage! ![Test Harness](.imgs/test-harness.png) ### Capturing HTTP Trace for new Fabric API calls Suppose you need to add payloads for a new fabric item type or an API call. The following script creates an HTTP snapshot that is stored in `http_trace.json.gz`, which is moved into `fabric-cicd/tests/fixtures` Update `item_type_in_scope` in the script with the item type you want to capture HTTP traffic for, then run: ```bash export FABRIC_WORKSPACE_ID="your-fabric-workspace-guid" uv run python devtools/debug_trace_deployment.py cp -f http_trace.json.gz tests/fixtures/http_trace.json.gz ``` You can validate the integration test works with the mock server with: ```bash uv run pytest -v -s --log-cli-level=INFO tests/test_integration_publish.py::test_publish_all_items_integration ``` ### Important Notes * The `http_trace.json` must be generated in one shot, i.e. the Mock Server is not guaranteed to incrementally process new lines added to `http_trace.json`. What that means is - you should capture as many items as possible in `debug_trace_deployment.py`, and use that payload in the tests. ### Troubleshooting * If a Unit Test fails on your branch but it wasn't failing on a previous branch, that means your branch contains new changes that have not been snapshotted. If you rever your branch to `main`, the tests should run green. Therefore, due to the logic/API call additions in your PR, you must regenerate the snapshot using the steps outlined above. * Copilot generated PRs that change API signatures will probably need human intervention to generate snapshots, as Copilot cannot make REST API calls to a real Fabric instance to regenerate the snapshot. In this situation, pull the Copilot generated PR locally and rerun the snapshot generation. ================================================ FILE: tests/fixtures/__init__.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Test fixtures and utilities.""" from fixtures.credentials import DummyTokenCredential, create_dummy_jwt from fixtures.mock_fabric_server import MOCK_SERVER_PORT, MockFabricServer __all__ = [ "MOCK_SERVER_PORT", "DummyTokenCredential", "MockFabricServer", "create_dummy_jwt", ] ================================================ FILE: tests/fixtures/credentials.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Test credentials and authentication utilities.""" import base64 import json import logging from datetime import datetime, timedelta, timezone from typing import Any from azure.core.credentials import AccessToken, TokenCredential def create_dummy_jwt(expiry_timestamp: int) -> str: """ Create a dummy JWT token for testing. Args: expiry_timestamp: Unix timestamp for token expiry Returns: A properly formatted JWT token string """ header = {"alg": "HS256", "typ": "JWT"} payload = { "exp": expiry_timestamp, "upn": "test@example.com", "aud": "https://api.fabric.microsoft.com", } header_b64 = base64.urlsafe_b64encode(json.dumps(header).encode()).decode().rstrip("=") payload_b64 = base64.urlsafe_b64encode(json.dumps(payload).encode()).decode().rstrip("=") signature = "dummy_signature" return f"{header_b64}.{payload_b64}.{signature}" class DummyTokenCredential(TokenCredential): """A static token credential for testing.""" def __init__(self, expiry_days: int = 365): """ Initialize static token credential. Args: expiry_days: Number of days until expiry. Defaults to 365. """ self.expiry = int((datetime.now(timezone.utc) + timedelta(days=expiry_days)).timestamp()) self._token = create_dummy_jwt(self.expiry) self.logger = logging.getLogger(__name__) def get_token(self, *scopes: str, **kwargs: Any) -> AccessToken: # noqa: ARG002 """Get the static access token.""" self.logger.debug(f"Static token credential - getting token for scopes: {scopes}") return AccessToken(self._token, self.expiry) def get_expire(self) -> int: """Get the token expiry timestamp.""" return self.expiry ================================================ FILE: tests/fixtures/mock_fabric_server.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """ Stateful Mock Fabric REST API server for integration testing. Provides transactionally correct responses via: - Content-based matching for POST/PATCH (by displayName + type) - Operation ID correlation for async 202 responses - State machine for long-running operations (Running -> Succeeded) - Generic fallback responses for unknown mutation routes """ import json import logging import re import threading from collections import defaultdict from http.server import BaseHTTPRequestHandler, HTTPServer from pathlib import Path from typing import Any, ClassVar, Optional from urllib.parse import urlparse from fabric_cicd._common._http_tracer import HTTPRequest, HTTPResponse logger = logging.getLogger(__name__) MOCK_SERVER_PORT = 8765 GUID_PATTERN = re.compile(r"[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}") OPERATION_PATTERN = re.compile(r"/v1/operations/([a-fA-F0-9-]+)(/result)?$") SKIP_HEADERS = { "content-length", "content-encoding", "transfer-encoding", "server", "date", "home-cluster-uri", "request-redirected", } class TraceIndex: """ Multi-index for trace lookup: by normalized route, by content (displayName, type), and by operation ID for async operation correlation. """ def __init__(self): self.by_route: dict[str, list[tuple[HTTPRequest, HTTPResponse]]] = defaultdict(list) self.by_content: dict[tuple[str, str, str], list[tuple[HTTPRequest, HTTPResponse]]] = defaultdict(list) self.by_operation: dict[str, dict[str, Any]] = defaultdict(lambda: {"post": None, "poll": [], "result": None}) @staticmethod def normalize_route(route: str) -> str: """Replace all GUIDs with {GUID} placeholder.""" return GUID_PATTERN.sub("{GUID}", route) @staticmethod def extract_content_key(body: Any, method: str) -> Optional[tuple[str, str, str]]: """Extract (displayName, type, method) from request body if present.""" if isinstance(body, dict) and body.get("displayName") and body.get("type"): return (body["displayName"], body["type"], method) return None def add_trace(self, request: HTTPRequest, response: HTTPResponse): """Index a trace by route, content, and operation ID.""" parsed = urlparse(request.url) route = parsed.path + (f"?{parsed.query}" if parsed.query else "") normalized_key = f"{request.method} {self.normalize_route(route)}" self.by_route[normalized_key].append((request, response)) if ( request.method in ("POST", "PATCH") and request.body and (content_key := self.extract_content_key(request.body, request.method)) ): self.by_content[content_key].append((request, response)) if (op_id := response.headers.get("x-ms-operation-id")) and response.status_code == 202: self.by_operation[op_id]["post"] = (request, response) if op_match := OPERATION_PATTERN.search(route): op_id, is_result = op_match.group(1), op_match.group(2) == "/result" if is_result: self.by_operation[op_id]["result"] = (request, response) else: self.by_operation[op_id]["poll"].append((request, response)) class MockFabricAPIHandler(BaseHTTPRequestHandler): """HTTP handler with stateful matching: content-based for items, state machine for operations.""" trace_index: ClassVar[TraceIndex] = TraceIndex() route_lock: ClassVar[threading.Lock] = threading.Lock() operation_poll_counts: ClassVar[dict[str, int]] = {} content_to_operation: ClassVar[dict[tuple[str, str], str]] = {} def log_message(self, format, *args): # noqa: A002 pass def do_GET(self): # noqa: N802 self._handle_request("GET") def do_POST(self): # noqa: N802 self._handle_request("POST") def do_PATCH(self): # noqa: N802 self._handle_request("PATCH") def do_DELETE(self): # noqa: N802 self._handle_request("DELETE") def _read_request_body(self) -> Optional[dict]: """Read and parse JSON request body.""" if content_length := self.headers.get("Content-Length"): try: return json.loads(self.rfile.read(int(content_length)).decode("utf-8")) except (json.JSONDecodeError, UnicodeDecodeError): pass return None def _handle_request(self, method: str): """Route request to appropriate handler, with fallback for unknown routes.""" route_key = f"{method} {self.path}" logger.info(f"Mock server received: {route_key}") request_body = self._read_request_body() if method in ("POST", "PATCH") else None response = self._find_matching_response(method, self.path, request_body) if response is None: if method in ("PATCH", "DELETE"): logger.info(f"No trace for {route_key}, returning 200 OK") response = HTTPResponse( status_code=200, headers={"Content-Type": "application/json"}, body={}, timestamp=None ) elif method == "POST": logger.info(f"No trace for {route_key}, returning 202 Accepted") response = HTTPResponse( status_code=202, headers={"Content-Type": "application/json"}, body={}, timestamp=None ) else: logger.warning(f"No trace data for {route_key}") self.send_error(404, f"No trace data found for {route_key}") return self._send_response(response, route_key) def _find_matching_response(self, method: str, route: str, request_body: Optional[dict]) -> Optional[HTTPResponse]: """Find response by: 1) operation polling, 2) content match, 3) route match.""" normalized_key = f"{method} {self.trace_index.normalize_route(route)}" if op_match := OPERATION_PATTERN.search(route): return self._handle_operation_request(op_match.group(1), op_match.group(2) == "/result") if ( method == "POST" and route.endswith("/items") and request_body and (content_key := self.trace_index.extract_content_key(request_body, method)) ): return self._handle_item_creation(content_key) if ( method == "POST" and "updateDefinition" in route and (traces := self.trace_index.by_route.get(normalized_key)) ): return traces[-1][1] if traces := self.trace_index.by_route.get(normalized_key): return traces[-1][1] return None def _handle_item_creation(self, content_key: tuple[str, str, str]) -> Optional[HTTPResponse]: """Match POST /items by (displayName, type), track operation for async responses.""" display_name, item_type, _ = content_key traces = self.trace_index.by_content.get(content_key, []) if not traces: logger.warning(f"No trace for item creation: {display_name} ({item_type})") return None for _, response in traces: if response.status_code in (200, 201, 202): if response.status_code == 202 and (op_id := response.headers.get("x-ms-operation-id")): with self.route_lock: self.content_to_operation[(display_name, item_type)] = op_id self.operation_poll_counts[op_id] = 0 logger.info(f"Tracking operation {op_id} for {display_name} ({item_type})") logger.info(f"Matched item creation: {display_name} ({item_type}) -> {response.status_code}") return response return traces[-1][1] def _handle_operation_request(self, operation_id: str, is_result: bool) -> Optional[HTTPResponse]: """ State machine for operation polling: first poll returns Running, subsequent return Succeeded. Result endpoint returns the created item details. """ op_data = self.trace_index.by_operation.get(operation_id) if not op_data: for _, known_data in self.trace_index.by_operation.items(): if known_data["post"] or known_data["result"]: op_data = known_data break if not op_data: logger.warning(f"No operation data for {operation_id}") return None if is_result: if op_data["result"]: logger.info(f"Returning operation result for {operation_id}") return op_data["result"][1] return None with self.route_lock: poll_count = self.operation_poll_counts.get(operation_id, 0) self.operation_poll_counts[operation_id] = poll_count + 1 poll_traces = op_data.get("poll", []) if not poll_traces: logger.warning(f"No poll traces for operation {operation_id}") return None target_status = "Running" if poll_count == 0 else "Succeeded" for _, response in poll_traces: if isinstance(response.body, dict) and response.body.get("status") == target_status: logger.info(f"Operation {operation_id} poll #{poll_count}: {target_status}") return response return poll_traces[-1][1] def _send_response(self, response: HTTPResponse, route_key: str): """Send HTTP response, rewriting Location headers for operations.""" self.send_response(response.status_code) body_bytes = json.dumps(response.body if isinstance(response.body, dict) else {}).encode() if isinstance(response.body, str) and response.body: body_bytes = response.body.encode() for name, value in response.headers.items(): lower = name.lower() if lower in ("x-ms-operation-id", "retry-after"): self.send_header(name, value) elif lower == "location" and "operations" in value and (op_match := OPERATION_PATTERN.search(value)): suffix = "/result" if op_match.group(2) == "/result" else "" self.send_header( "Location", f"http://127.0.0.1:{MOCK_SERVER_PORT}/v1/operations/{op_match.group(1)}{suffix}" ) elif lower not in SKIP_HEADERS: self.send_header(name, value) self.send_header("Content-Length", len(body_bytes)) self.end_headers() self.wfile.write(body_bytes) logger.debug(f"Responded to {route_key}: {response.status_code}") @classmethod def load_trace_data(cls, trace_file: Path): """Load trace data from JSON file and build indices.""" cls.trace_index = TraceIndex() cls.operation_poll_counts.clear() cls.content_to_operation.clear() with trace_file.open("r") as f: data = json.load(f) loaded = 0 for trace in data.get("traces", []): try: req, resp = trace.get("request"), trace.get("response") if not req or not resp: continue cls.trace_index.add_trace( HTTPRequest( req.get("method", ""), req.get("url", ""), req.get("headers", {}), req.get("body"), req.get("timestamp"), ), HTTPResponse( resp.get("status_code", 200), resp.get("headers", {}), resp.get("body"), resp.get("timestamp") ), ) loaded += 1 except Exception as e: logger.warning(f"Failed to parse trace: {e}") logger.info( f"Loaded {loaded} traces (routes={len(cls.trace_index.by_route)}, content={len(cls.trace_index.by_content)}, ops={len(cls.trace_index.by_operation)})" ) class MockFabricServer: """Mock Fabric API server for testing.""" HTTP_TRACE_FILE = "http_trace.json.gz" def __init__(self, trace_file: Path, port: int = MOCK_SERVER_PORT): self.port = port self.trace_file = trace_file self.server: Optional[HTTPServer] = None self.server_thread: Optional[threading.Thread] = None def start(self): """Start the mock server in a background thread.""" MockFabricAPIHandler.load_trace_data(self.trace_file) self.server = HTTPServer(("127.0.0.1", self.port), MockFabricAPIHandler) self.server_thread = threading.Thread(target=self.server.serve_forever, daemon=True) self.server_thread.start() logger.info(f"Mock Fabric API server started on http://127.0.0.1:{self.port}") def stop(self): """Stop the mock server.""" if self.server: self.server.shutdown() self.server.server_close() if self.server_thread: self.server_thread.join(timeout=5) logger.info("Mock Fabric API server stopped") ================================================ FILE: tests/test__check_utils.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. import json import pytest from fabric_cicd._common._check_utils import check_file_type, check_valid_json_content, check_valid_yaml_content @pytest.fixture def text_file(tmp_path): file_path = tmp_path / "test.txt" file_path.write_text("sample text") return file_path @pytest.fixture def binary_file(tmp_path): file_path = tmp_path / "test.bin" file_path.write_bytes( b"PK\x03\x04\x14\x00\x00\x00\x08\x00\x00\x00!\x00\xb7\xac\xce\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" ) return file_path @pytest.fixture def image_file(tmp_path): file_path = tmp_path / "test.png" file_path.write_bytes(b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01") return file_path def test_check_file_type_text(text_file): assert check_file_type(text_file) == "text" def test_check_file_type_binary(binary_file): assert check_file_type(binary_file) == "binary" def test_check_file_type_image(image_file): assert check_file_type(image_file) == "image" @pytest.fixture def real_schedules_file(tmp_path): """Create a realistic .schedules file with exact structure like fabric-cicd uses.""" # Create a DataPipeline directory structure pipeline_dir = tmp_path / "Test Pipeline.DataPipeline" pipeline_dir.mkdir() schedules_file = pipeline_dir / ".schedules" schedules_content = { "schedules": [ { "jobType": "Execute", "enabled": True, "cronExpression": "0 0 12 * * ?", "timeZone": "UTC", "description": "Daily execution at noon", }, { "jobType": "Refresh", "enabled": False, "cronExpression": "0 0 6 * * ?", "timeZone": "UTC", "description": "Morning refresh", }, ] } schedules_file.write_text(json.dumps(schedules_content, indent=2)) return schedules_file def test_schedules_file_json_validation_and_structure(real_schedules_file): """Test that .schedules files are properly validated and contain expected structure.""" # Test that check_valid_json_content correctly identifies .schedules content as valid JSON content = real_schedules_file.read_text(encoding="utf-8") assert check_valid_json_content(content) is True # Verify the file has the expected structure for key_value_replace data = json.loads(content) # Verify the structure matches what the JSONPath expression expects assert "schedules" in data assert isinstance(data["schedules"], list) assert len(data["schedules"]) >= 1 # Find Execute job and verify it has enabled field execute_jobs = [schedule for schedule in data["schedules"] if schedule.get("jobType") == "Execute"] assert len(execute_jobs) >= 1 execute_job = execute_jobs[0] assert "enabled" in execute_job assert isinstance(execute_job["enabled"], bool) # Verify file path ends with .schedules assert real_schedules_file.name == ".schedules" assert str(real_schedules_file).endswith(".schedules") def test_schedules_file_jsonpath_compatibility(real_schedules_file): """Test that .schedules files work with the specific JSONPath expression used in parameter.yml.""" try: from jsonpath_ng.ext import parse except ImportError: pytest.skip("jsonpath_ng not available for testing") # Read and parse the .schedules file content = real_schedules_file.read_text(encoding="utf-8") data = json.loads(content) # Test the exact JSONPath expression from the parameter.yml jsonpath_expr = parse('$.schedules[?(@.jobType=="Execute")].enabled') matches = [match.value for match in jsonpath_expr.find(data)] # Should find at least one enabled field from Execute jobs assert len(matches) >= 1 assert all(isinstance(match, bool) for match in matches) # Verify we can access the specific value that would be replaced first_match = matches[0] assert first_match is True # Our test data has enabled=True def test_real_sample_schedules_file(): """Test that the actual sample .schedules file works with our function.""" from pathlib import Path schedules_file = Path("sample/workspace/Run Hello World.DataPipeline/.schedules") # Skip if sample file doesn't exist (optional test) if not schedules_file.exists(): pytest.skip("Sample .schedules file not found") # Test that our function works with the real file content content = schedules_file.read_text(encoding="utf-8") assert check_valid_json_content(content) is True # Verify the structure contains what we expect data = json.loads(content) assert "schedules" in data assert isinstance(data["schedules"], list) def test_check_valid_json_content_with_valid_json(): """Test check_valid_json_content with valid JSON string.""" valid_json = '{"key": "value", "number": 123, "boolean": true}' assert check_valid_json_content(valid_json) is True def test_check_valid_json_content_with_invalid_json(): """Test check_valid_json_content with invalid JSON string.""" invalid_json = '{"key": "value" invalid json}' assert check_valid_json_content(invalid_json) is False def test_check_valid_json_content_with_empty_string(): """Test check_valid_json_content with empty string.""" assert check_valid_json_content("") is False def test_check_valid_json_content_with_schedules_structure(): """Test check_valid_json_content with realistic schedules JSON structure.""" schedules_json = json.dumps({ "schedules": [{"jobType": "Execute", "enabled": True, "cronExpression": "0 0 12 * * ?"}] }) assert check_valid_json_content(schedules_json) is True def test_check_valid_yaml_content_with_valid_yaml(): """Test check_valid_yaml_content with valid YAML string.""" valid_yaml = """ server: host: localhost port: 8080 """ assert check_valid_yaml_content(valid_yaml) is True def test_check_valid_yaml_content_with_invalid_yaml(): """Test check_valid_yaml_content with invalid YAML string.""" invalid_yaml = "invalid: yaml: [unclosed" assert check_valid_yaml_content(invalid_yaml) is False def test_check_valid_yaml_content_with_empty_string(): """Test check_valid_yaml_content with empty string.""" # Empty string parses as None, not a structured YAML mapping/sequence assert check_valid_yaml_content("") is False def test_check_valid_yaml_content_with_spark_compute_structure(): """Test check_valid_yaml_content with realistic SparkCompute.yml structure.""" spark_compute_yaml = """ enable_native_execution_engine: false driver_cores: 8 driver_memory: 56g executor_cores: 8 executor_memory: 56g dynamic_executor_allocation: enabled: true min_executors: 1 max_executors: 9 runtime_version: "1.2" """ assert check_valid_yaml_content(spark_compute_yaml) is True def test_check_valid_yaml_content_with_complex_structure(): """Test check_valid_yaml_content with complex nested YAML structure.""" complex_yaml = """ config: database: host: localhost port: 5432 credentials: username: admin password: secret features: - name: feature1 enabled: true - name: feature2 enabled: false limits: max_connections: 100 timeout: 30.5 """ assert check_valid_yaml_content(complex_yaml) is True def test_check_valid_yaml_content_vs_json_content(): """Test that JSON is also valid YAML (JSON is a subset of YAML).""" json_content = '{"key": "value", "number": 123}' # JSON is valid YAML assert check_valid_yaml_content(json_content) is True assert check_valid_json_content(json_content) is True # YAML-only content is not valid JSON yaml_only_content = """ key: value number: 123 """ assert check_valid_yaml_content(yaml_only_content) is True assert check_valid_json_content(yaml_only_content) is False def test_check_valid_yaml_content_with_notebook_py_content(): """Test that notebook .py files with # comments are not treated as valid YAML.""" notebook_content = """# Fabric notebook source # METADATA ******************** # META { # META "kernel_info": { # META "name": "synapse_pyspark" # META } # META } # CELL ******************** print("hello world") """ assert check_valid_yaml_content(notebook_content) is False def test_check_valid_yaml_content_with_kql_content(): """Test that KQL script files are not treated as valid YAML.""" kql_content = """// KQL script // Use management commands to configure your database items. .create-merge table YellowTaxi (vendorID:string, tpepPickupDateTime:datetime) .create-or-alter table YellowTaxi ingestion json mapping 'YellowTaxi_mapping' """ assert check_valid_yaml_content(kql_content) is False def test_check_valid_yaml_content_with_plain_python_content(): """Test that plain Python files (e.g., SparkJobDefinition) are not treated as valid YAML.""" python_content = """from pyspark.sql import SparkSession import logging logger = logging.getLogger(__name__) if __name__ == "__main__": spark = SparkSession.builder.appName("test").getOrCreate() df = spark.read.format("delta").load("Tables/my_table") df.show() """ assert check_valid_yaml_content(python_content) is False ================================================ FILE: tests/test__fabric_endpoint.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. import datetime import time from unittest.mock import Mock import pytest from azure.core.exceptions import ClientAuthenticationError from fabric_cicd import constants from fabric_cicd._common._exceptions import InvokeError, TokenError from fabric_cicd._common._fabric_endpoint import FabricEndpoint, _format_invoke_log, _handle_response class DummyLogger: def __init__(self): self.messages = [] def info(self, message): self.messages.append(message) def debug(self, message): self.messages.append(message) class DummyCredential: def __init__(self, token, expires_on=9999999999): self.token = token self.expires_on = expires_on self.raise_exception = None def get_token(self, *_, **__): if self.raise_exception: raise self.raise_exception return Mock(token=self.token, expires_on=self.expires_on) @pytest.fixture def setup_mocks(monkeypatch, mocker): dl = DummyLogger() mock_logger = mocker.Mock() mock_logger.isEnabledFor.return_value = True mock_logger.info.side_effect = dl.info mock_logger.debug.side_effect = dl.debug monkeypatch.setattr("fabric_cicd._common._fabric_endpoint.logger", mock_logger) mock_requests = mocker.patch("requests.request") return dl, mock_requests def generate_mock_token(): return "mock_token_value" def test_integration(setup_mocks): """Test integration of FabricEndpoint for GET request.""" _, mock_requests = setup_mocks mock_requests.return_value = Mock( status_code=200, headers={"Content-Type": "application/json"}, json=Mock(return_value={}) ) mock_token_credential = Mock() mock_token_credential.get_token.return_value = Mock(token=generate_mock_token(), expires_on=9999999999) endpoint = FabricEndpoint(token_credential=mock_token_credential) response = endpoint.invoke("GET", "http://example.com") assert response["status_code"] == 200 def test_performance(setup_mocks): """Test that _handle_response completes quickly under long-running simulation.""" _, _mock_requests = setup_mocks response = Mock(status_code=200, headers={}, json=Mock(return_value={"status": "Succeeded"})) start_time = time.time() _handle_response( response=response, method="GET", url="old", body="{}", long_running=True, iteration_count=2, ) end_time = time.time() assert (end_time - start_time) < 1 # Ensure the function completes within 1 second @pytest.mark.parametrize( ("method", "url", "body", "files"), [ ("GET", "http://example.com", "{}", None), ("POST", "http://example.com", "{}", {"file": "test.txt"}), ], ids=["invoke", "invoke_with_files"], ) def test_invoke(setup_mocks, method, url, body, files): """Test FabricEndpoint invoke method success + with optional files.""" _, mock_requests = setup_mocks mock_requests.return_value = Mock( status_code=200, headers={"Content-Type": "application/json"}, json=Mock(return_value={}) ) mock_token_credential = Mock() mock_token_credential.get_token.return_value = Mock(token=generate_mock_token(), expires_on=9999999999) endpoint = FabricEndpoint(token_credential=mock_token_credential) response = endpoint.invoke(method, url, body, files) assert response["status_code"] == 200 def test_invoke_token_expired(setup_mocks, monkeypatch): """Test invoking endpoint when the AAD token is expired and refreshed.""" dl, mock_requests = setup_mocks mock_requests.side_effect = [ Mock(status_code=401, headers={"x-ms-public-api-error-code": "TokenExpired"}), Mock(status_code=200, headers={"Content-Type": "application/json"}, json=Mock(return_value={})), ] mock_token_credential = Mock() mock_token_credential.get_token.return_value = Mock(token=generate_mock_token(), expires_on=9999999999) endpoint = FabricEndpoint(token_credential=mock_token_credential) endpoint.aad_token_expiration = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(seconds=1) endpoint._refresh_token = Mock() monkeypatch.setattr("fabric_cicd._common._fabric_endpoint._format_invoke_log", lambda *_, **__: "") response = endpoint.invoke("GET", "http://example.com") assert f"{constants.INDENT}AAD token expired. Refreshing token." in dl.messages assert response["status_code"] == 200 def test_invoke_exception(setup_mocks): """Test invoking endpoint when the AAD token is expired and refreshed.""" _, mock_requests = setup_mocks mock_requests.side_effect = Exception("Test exception") mock_token_credential = Mock() mock_token_credential.get_token.return_value = Mock(token=generate_mock_token(), expires_on=9999999999) endpoint = FabricEndpoint(token_credential=mock_token_credential) with pytest.raises(InvokeError): endpoint.invoke("GET", "http://example.com") def test_invoke_poll_long_running_false_with_202(setup_mocks): """Test invoke method with poll_long_running=False exits early on 202 response.""" _, mock_requests = setup_mocks mock_requests.return_value = Mock( status_code=202, headers={"Content-Type": "application/json", "Location": "http://example.com/status"}, json=Mock(return_value={}), ) mock_token_credential = Mock() mock_token_credential.get_token.return_value = Mock(token=generate_mock_token(), expires_on=9999999999) endpoint = FabricEndpoint(token_credential=mock_token_credential) response = endpoint.invoke("POST", "http://example.com", poll_long_running=False) # Should exit immediately without polling assert response["status_code"] == 202 assert mock_requests.call_count == 1 # Only one request, no polling def test_invoke_poll_long_running_true_with_202(setup_mocks, monkeypatch): """Test invoke method with poll_long_running=True polls on 202 response.""" _, mock_requests = setup_mocks # First call returns 202 with Location header, second call returns 200 with Succeeded status mock_requests.side_effect = [ Mock( status_code=202, headers={"Content-Type": "application/json", "Location": "http://example.com/status"}, json=Mock(return_value={}), text="{}", ), Mock( status_code=200, headers={"Content-Type": "application/json"}, json=Mock(return_value={"status": "Succeeded"}), text='{"status": "Succeeded"}', ), ] mock_token_credential = Mock() mock_token_credential.get_token.return_value = Mock(token=generate_mock_token(), expires_on=9999999999) endpoint = FabricEndpoint(token_credential=mock_token_credential) # Mock time.sleep to avoid delays in tests monkeypatch.setattr("time.sleep", lambda _: None) response = endpoint.invoke("POST", "http://example.com", poll_long_running=True) # Should poll and return final status assert response["status_code"] == 200 assert mock_requests.call_count == 2 # Initial request + polling request def test_invoke_poll_long_running_default_with_202(setup_mocks, monkeypatch): """Test invoke method with default poll_long_running (True) polls on 202 response.""" _, mock_requests = setup_mocks # First call returns 202 with Location header, second call returns 200 with Succeeded status mock_requests.side_effect = [ Mock( status_code=202, headers={"Content-Type": "application/json", "Location": "http://example.com/status"}, json=Mock(return_value={}), text="{}", ), Mock( status_code=200, headers={"Content-Type": "application/json"}, json=Mock(return_value={"status": "Succeeded"}), text='{"status": "Succeeded"}', ), ] mock_token_credential = Mock() mock_token_credential.get_token.return_value = Mock(token=generate_mock_token(), expires_on=9999999999) endpoint = FabricEndpoint(token_credential=mock_token_credential) # Mock time.sleep to avoid delays in tests monkeypatch.setattr("time.sleep", lambda _: None) # Don't pass poll_long_running, should default to True response = endpoint.invoke("POST", "http://example.com") # Should poll and return final status assert response["status_code"] == 200 assert mock_requests.call_count == 2 # Initial request + polling request def test_refresh_token(setup_mocks): """Test refreshing token sets token and expiration from AccessToken.""" _dl, _mock_requests = setup_mocks mock_token_credential = Mock() mock_token_credential.get_token.return_value = Mock(token="test_token", expires_on=9999999999) endpoint = FabricEndpoint(token_credential=mock_token_credential) assert endpoint.aad_token == "test_token" assert endpoint.aad_token_expiration == datetime.datetime.fromtimestamp(9999999999, tz=datetime.timezone.utc) @pytest.mark.parametrize( ("raise_exception", "expected_msg"), [ (ClientAuthenticationError("Auth failed"), "Failed to acquire AAD token. Auth failed"), (Exception("Unexpected error"), "An unexpected error occurred when generating the AAD token. Unexpected error"), ], ids=["auth_error", "unexpected_exception"], ) def test_refresh_token_exceptions(raise_exception, expected_msg): """Test token refresh exception handling for authentication failures.""" credential = DummyCredential("irrelevant") credential.raise_exception = raise_exception with pytest.raises(TokenError, match=expected_msg): FabricEndpoint(token_credential=credential) @pytest.mark.parametrize( ( "status_code", "request_method", "expected_long_running", "expected_exit_loop", "input_long_running", "input_iteration_count", "response_header", "response_json", ), [ (200, "POST", False, True, False, 1, {}, {}), (202, "POST", True, False, False, 1, {"Retry-After": 20, "Location": "new"}, {}), (200, "GET", True, False, True, 2, {"Retry-After": 20, "Location": "old"}, {"status": "Running"}), (200, "GET", False, True, True, 2, {}, {"status": "Succeeded"}), (200, "GET", False, False, True, 2, {"Retry-After": 20, "Location": "old"}, {"status": "Succeeded"}), ], ids=[ "success", "long_running_redirect", "long_running_running", "long_running_success", "long_running_success_with_result", ], ) def test_handle_response( status_code, request_method, expected_long_running, expected_exit_loop, input_long_running, input_iteration_count, response_header, response_json, ): """Test _handle_response behavior for various HTTP responses and long-running operations.""" response = Mock(status_code=status_code, headers=response_header, json=Mock(return_value=response_json)) exit_loop, _method, _url, _body, long_running = _handle_response( response=response, method=request_method, url="old", body="{}", long_running=input_long_running, iteration_count=input_iteration_count, ) assert exit_loop == expected_exit_loop assert long_running == expected_long_running @pytest.mark.parametrize( ("exception_match", "response_json"), [ ( "[Operation failed].*", {"status": "Failed", "error": {"errorCode": "SampleErrorCode", "message": "Sample failure message"}}, ), ("[Operation is in an undefined state].*", {"status": "Undefined"}), ], ids=["failed", "undefined"], ) def test_handle_response_longrunning_exception(exception_match, response_json): """Test _handle_response raises exception for longrunning failure conditions.""" response = Mock(status_code=200, headers={}, json=Mock(return_value=response_json)) with pytest.raises(Exception, match=exception_match): _handle_response( response=response, method="GET", url="old", body="{}", long_running=True, iteration_count=2, ) @pytest.mark.parametrize( ( "status_code", "input_iteration_count", "input_long_running", "response_header", "return_value", "exception_match", "max_duration", "start_time", ), [ ( 401, 1, False, {"x-ms-public-api-error-code": "Unauthorized"}, {}, "The executing identity is not authorized to call GET on 'http://example.com'.", None, None, ), ( 400, 1, False, {"x-ms-public-api-error-code": "PrincipalTypeNotSupported"}, {}, "The executing principal type is not supported to call GET on 'http://example.com'.", None, None, ), ( 400, 1, False, {"x-ms-public-api-error-code": "PrincipalTypeNotSupported"}, {"message": "Test Libabry is not present in the environment."}, "Deployment attempted to remove a library that is not present in the environment. ", None, None, ), ( 500, 5, False, {"Content-Type": "application/json"}, {"message": "Internal Server Error"}, r"Maximum execution duration \(0 seconds\) exceeded", 0, 0.0, ), (429, 5, True, {"Retry-After": "10"}, {}, r"Maximum execution duration \(0 seconds\) exceeded", 0, 0.0), ( 200, 5, True, {}, {"status": "Running"}, r"Maximum execution duration \(0 seconds\) exceeded", 0, 0.0, ), ], ids=[ "unauthorized", "principal_type_not_supported", "failed_library_removal", "retry_500", "retry_429", "long_running_timeout", ], ) def test_handle_response_exceptions( status_code, input_iteration_count, input_long_running, response_header, return_value, exception_match, max_duration, start_time, ): """Test _handle_response raises appropriate exceptions based on response error codes.""" response = Mock(status_code=status_code, headers=response_header, json=Mock(return_value=return_value)) with pytest.raises(Exception, match=exception_match): _handle_response( response=response, method="GET", url="http://example.com", body="{}", long_running=input_long_running, iteration_count=input_iteration_count, max_duration=max_duration, start_time=start_time, ) def test_handle_response_feature_not_available(): """Test _handle_response for feature not available""" response = Mock(status_code=403, reason="FeatureNotAvailable") with pytest.raises(Exception, match=r"Item type not supported. Description: FeatureNotAvailable"): _handle_response( response=response, method="GET", url="http://example.com", body="{}", long_running=False, iteration_count=1, ) def test_handle_response_item_display_name_already_in_use(setup_mocks, monkeypatch): """ Test _handle_response logs a retry message when item display name is already in use. Mocks time.sleep to avoid actual test execution delays. """ import time dl, _mock_requests = setup_mocks monkeypatch.setattr("time.sleep", lambda _: None) response = Mock(status_code=400, headers={"x-ms-public-api-error-code": "ItemDisplayNameNotAvailableYet"}) _handle_response(response, "GET", "http://example.com", "{}", False, 1, max_duration=300, start_time=time.time()) expected = f"{constants.INDENT}Item name is reserved. Checking again in 60 seconds (Attempt 1)..." assert dl.messages == [expected] def test_handle_response_environment_libraries_not_found(setup_mocks): """Test _handle_response exits loop when environment libraries are not found (404).""" _, _mock_requests = setup_mocks response = Mock(status_code=404, headers={"x-ms-public-api-error-code": "EnvironmentLibrariesNotFound"}) exit_loop, _method, _url, _body, long_running = _handle_response( response=response, method="GET", url="http://example.com", body="{}", long_running=False, iteration_count=1, ) assert exit_loop is True assert long_running is False def test_format_invoke_log(): """Test formatting of the invoke log message.""" response = Mock(status_code=200, headers={"Content-Type": "application/json"}, json=Mock(return_value={})) log_message = _format_invoke_log(response, "GET", "http://example.com", "{}") assert "Method: GET" in log_message assert "URL: http://example.com" in log_message ================================================ FILE: tests/test__file.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. import base64 from pathlib import Path import pytest from fabric_cicd._common._file import File SAMPLE_IMAGE_DATA = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01" SAMPLE_TEXT_DATA = "sample text" @pytest.fixture def text_file(tmp_path): item_path = tmp_path / "workspace/ABC.SemanticModel" file_path = item_path / "definition/tables/Table.tmdl" file_path.parent.mkdir(parents=True, exist_ok=True) # Ensure the parent directories are created file_path.write_text(SAMPLE_TEXT_DATA) return File(item_path=item_path, file_path=file_path) @pytest.fixture def image_file(tmp_path): item_path = tmp_path / "workspace/ABC.Report" file_path = item_path / "StaticResources/RegisteredResources/image.png" file_path.parent.mkdir(parents=True, exist_ok=True) # Ensure the parent directories are created file_path.write_bytes(SAMPLE_IMAGE_DATA) return File(item_path=item_path, file_path=file_path) def test_file_text_initialization(text_file): assert text_file.name == "Table.tmdl" assert text_file.contents == SAMPLE_TEXT_DATA assert text_file.relative_path == "definition/tables/Table.tmdl" def test_file_text_payload(text_file): expected_payload = base64.b64encode(SAMPLE_TEXT_DATA.encode("utf-8")).decode("utf-8") assert text_file.base64_payload == { "path": "definition/tables/Table.tmdl", "payload": expected_payload, "payloadType": "InlineBase64", } def test_file_text_set_contents(text_file): text_file.contents = "New contents" assert text_file.contents == "New contents" def test_file_image_immutable_fields(image_file): with pytest.raises(AttributeError): image_file.item_path = Path("/new/path") with pytest.raises(AttributeError): image_file.file_path = Path("/new/path") with pytest.raises(AttributeError): image_file.contents = "new contents" def test_file_image_payload(image_file): expected_payload = base64.b64encode(SAMPLE_IMAGE_DATA).decode("utf-8") assert image_file.base64_payload == { "path": "StaticResources/RegisteredResources/image.png", "payload": expected_payload, "payloadType": "InlineBase64", } def test_file_text_special_characters(tmp_path): special_text = ( "Greek: a, β, y, δ, ε, ζ\n" "Latin: æ, ø, å, ß, é, ñ, ü\n" "Cyrillic: Ж, Д, и, ю\n" "Arabic: مرحبا, سلام\n" "Chinese: 你好, 世界\n" "Emoji: 😊, 🚀, 🌟\n" "Symbols: ©, ®, ™, €, £, ¥, ∞, ≠, ≤, ≥" ) item_path = tmp_path / "workspace/SpecialTextModel" file_path = item_path / "definition/special.txt" file_path.parent.mkdir(parents=True, exist_ok=True) file_path.write_text(special_text, encoding="utf-8") file_obj = File(item_path=item_path, file_path=file_path) expected_payload = base64.b64encode(special_text.encode("utf-8")).decode("utf-8") assert file_obj.contents == special_text assert file_obj.base64_payload == { "path": "definition/special.txt", "payload": expected_payload, "payloadType": "InlineBase64", } ================================================ FILE: tests/test_config_validator.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Unit tests for ConfigValidator class.""" from pathlib import Path from unittest.mock import patch import pytest import yaml from fabric_cicd import constants from fabric_cicd._common._config_validator import ConfigValidationError, ConfigValidator class TestConfigValidator: """Unit tests for ConfigValidator class.""" def setup_method(self): """Set up for each test method.""" self.validator = ConfigValidator() def test_init(self): """Test ConfigValidator initialization.""" assert self.validator.errors == [] assert self.validator.config is None assert self.validator.config_path is None assert self.validator.environment is None def test_validate_file_existence_valid_file(self, tmp_path): """Test _validate_file_existence with valid file.""" config_file = tmp_path / "config.yaml" config_file.write_text("test: value") result = self.validator._validate_file_existence(str(config_file)) assert result == config_file.resolve() assert self.validator.errors == [] def test_validate_file_existence_missing_file(self): """Test _validate_file_existence with missing file.""" result = self.validator._validate_file_existence("nonexistent.yaml") assert result is None assert len(self.validator.errors) == 1 assert ( constants.CONFIG_VALIDATION_MSGS["file"]["not_found"].format("nonexistent.yaml") in self.validator.errors[0] ) @pytest.mark.parametrize("path_value", ["", None]) def test_validate_file_existence_empty_or_none_path(self, path_value): """Test _validate_file_existence with empty or None path.""" result = self.validator._validate_file_existence(path_value) assert result is None assert len(self.validator.errors) == 1 assert constants.CONFIG_VALIDATION_MSGS["file"]["path_empty"] in self.validator.errors[0] def test_validate_file_existence_directory_instead_of_file(self, tmp_path): """Test _validate_file_existence with directory instead of file.""" result = self.validator._validate_file_existence(str(tmp_path)) assert result is None assert len(self.validator.errors) == 1 assert constants.CONFIG_VALIDATION_MSGS["file"]["not_file"].format(str(tmp_path)) in self.validator.errors[0] def test_validate_file_existence_invalid_path_os_error(self): """Test _validate_file_existence with a path that triggers OSError.""" with patch("fabric_cicd._common._config_validator.Path.resolve", side_effect=OSError("bad path")): result = self.validator._validate_file_existence("some_path") assert result is None assert len(self.validator.errors) == 1 assert ( constants.CONFIG_VALIDATION_MSGS["file"]["invalid_path"].format("some_path", "bad path") in self.validator.errors[0] ) def test_validate_yaml_content_valid_yaml(self, tmp_path): """Test _validate_yaml_content with valid YAML.""" config_file = tmp_path / "config.yaml" config_data = {"core": {"workspace_id": "test-id"}} config_file.write_text(yaml.dump(config_data)) self.validator.config_path = config_file result = self.validator._validate_yaml_content(config_file) assert result == config_data assert self.validator.errors == [] def test_validate_yaml_content_invalid_yaml(self, tmp_path): """Test _validate_yaml_content with invalid YAML syntax.""" config_file = tmp_path / "config.yaml" config_file.write_text("invalid: yaml: content: [") self.validator.config_path = config_file result = self.validator._validate_yaml_content(config_file) assert result is None assert len(self.validator.errors) == 1 # We can't test the exact error message as it includes the specific parse error assert constants.CONFIG_VALIDATION_MSGS["file"]["yaml_syntax"].split(":")[0] in self.validator.errors[0] def test_validate_yaml_content_unicode_decode_error(self, tmp_path): """Test _validate_yaml_content with file that triggers UnicodeDecodeError.""" config_file = tmp_path / "config.yaml" config_file.write_bytes(b"\x80\x81\x82") # Invalid UTF-8 self.validator.config_path = config_file result = self.validator._validate_yaml_content(config_file) assert result is None assert len(self.validator.errors) == 1 assert constants.CONFIG_VALIDATION_MSGS["file"]["encoding_error"].split(":")[0] in self.validator.errors[0] def test_validate_yaml_content_permission_error(self, tmp_path): """Test _validate_yaml_content with file that triggers PermissionError.""" config_file = tmp_path / "config.yaml" config_file.write_text("valid: yaml") self.validator.config_path = config_file with patch("pathlib.Path.open", side_effect=PermissionError("access denied")): result = self.validator._validate_yaml_content(config_file) assert result is None assert len(self.validator.errors) == 1 assert constants.CONFIG_VALIDATION_MSGS["file"]["permission_denied"].split(":")[0] in self.validator.errors[0] def test_validate_yaml_content_unexpected_error(self, tmp_path): """Test _validate_yaml_content with file that triggers unexpected error.""" config_file = tmp_path / "config.yaml" config_file.write_text("valid: yaml") self.validator.config_path = config_file with patch("pathlib.Path.open", side_effect=RuntimeError("unexpected")): result = self.validator._validate_yaml_content(config_file) assert result is None assert len(self.validator.errors) == 1 assert constants.CONFIG_VALIDATION_MSGS["file"]["unexpected_error"].split(":")[0] in self.validator.errors[0] def test_validate_yaml_content_non_dict_yaml(self, tmp_path): """Test _validate_yaml_content with non-dictionary YAML.""" config_file = tmp_path / "config.yaml" config_file.write_text("- item1\n- item2") self.validator.config_path = config_file result = self.validator._validate_yaml_content(config_file) assert result is None assert len(self.validator.errors) == 1 assert constants.CONFIG_VALIDATION_MSGS["file"]["not_dict"].format("list") in self.validator.errors[0] def test_validate_yaml_content_none_path(self): """Test _validate_yaml_content with None path.""" result = self.validator._validate_yaml_content(None) assert result is None assert self.validator.errors == [] # Error should already be added by file existence check def test_validate_yaml_content_empty_file(self, tmp_path): """Test _validate_yaml_content with empty file.""" config_file = tmp_path / "config.yaml" config_file.write_text("") self.validator.config_path = config_file result = self.validator._validate_yaml_content(config_file) assert result is None assert len(self.validator.errors) == 1 assert constants.CONFIG_VALIDATION_MSGS["file"]["empty_file"] in self.validator.errors[0] def test_validate_config_structure_valid(self): """Test _validate_config_structure with valid config.""" self.validator.config = {"core": {"workspace_id": "test-id"}} self.validator._validate_config_structure() assert self.validator.errors == [] @pytest.mark.parametrize("config_value", [["not", "a", "dict"], None]) def test_validate_config_structure_non_dict_or_none(self, config_value): """Test _validate_config_structure with non-dictionary or None config.""" self.validator.config = config_value self.validator._validate_config_structure() # The structure validation doesn't add errors for non-dict/None configs # as this is handled by YAML content validation assert self.validator.errors == [] def test_validate_config_structure_missing_core(self): """Test _validate_config_structure with config missing 'core' section.""" self.validator.config = {"features": ["f1"]} self.validator._validate_config_structure() assert len(self.validator.errors) == 1 assert constants.CONFIG_VALIDATION_MSGS["structure"]["missing_core"] in self.validator.errors[0] def test_validate_config_structure_core_not_dict(self): """Test _validate_config_structure with 'core' as non-dict.""" self.validator.config = {"core": "not a dict"} self.validator._validate_config_structure() assert len(self.validator.errors) == 1 assert constants.CONFIG_VALIDATION_MSGS["structure"]["core_not_dict"].format("str") in self.validator.errors[0] def test_validate_workspace_field_valid_string(self): """Test _validate_workspace_field with valid string.""" core = {"workspace": "test-workspace"} result = self.validator._validate_workspace_field(core, "workspace") assert result is True assert self.validator.errors == [] def test_validate_workspace_field_valid_workspace_id_guid(self): """Test _validate_workspace_field with valid workspace_id GUID.""" core = {"workspace_id": "8b6e2c7a-4c1f-4e3a-9b2e-7d8f2e1a6c3b"} result = self.validator._validate_workspace_field(core, "workspace_id") assert result is True assert self.validator.errors == [] def test_validate_workspace_field_invalid_workspace_id_guid(self): """Test _validate_workspace_field with invalid workspace_id GUID format.""" core = {"workspace_id": "invalid-guid-format"} result = self.validator._validate_workspace_field(core, "workspace_id") assert result is False assert len(self.validator.errors) == 1 assert "must be a valid GUID format" in self.validator.errors[0] def test_validate_workspace_field_valid_dict(self): """Test _validate_workspace_field with valid environment mapping.""" core = {"workspace": {"dev": "dev-workspace", "prod": "prod-workspace"}} result = self.validator._validate_workspace_field(core, "workspace") assert result is True assert self.validator.errors == [] def test_validate_workspace_field_valid_workspace_id_dict(self): """Test _validate_workspace_field with valid workspace_id environment mapping.""" core = { "workspace_id": { "dev": "8b6e2c7a-4c1f-4e3a-9b2e-7d8f2e1a6c3b", "prod": "2f4b9e8d-1a7c-4d3e-b8e2-5c9f7a2d4e1b", } } result = self.validator._validate_workspace_field(core, "workspace_id") assert result is True assert self.validator.errors == [] def test_validate_workspace_field_invalid_workspace_id_dict(self): """Test _validate_workspace_field with invalid workspace_id GUID in environment mapping.""" core = {"workspace_id": {"dev": "valid-8b6e2c7a-4c1f-4e3a-9b2e-7d8f2e1a6c3b", "prod": "invalid-guid"}} result = self.validator._validate_workspace_field(core, "workspace_id") assert result is False assert len(self.validator.errors) == 2 # One for each invalid GUID assert "must be a valid GUID format" in self.validator.errors[0] assert "must be a valid GUID format" in self.validator.errors[1] def test_validate_workspace_field_missing(self): """Test _validate_workspace_field with missing field.""" core = {} result = self.validator._validate_workspace_field(core, "workspace_id") assert result is False assert self.validator.errors == [] def test_validate_workspace_field_empty_string(self): """Test _validate_workspace_field with empty/whitespace string.""" core = {"workspace_id": " "} result = self.validator._validate_workspace_field(core, "workspace_id") assert result is False assert len(self.validator.errors) == 1 assert ( constants.CONFIG_VALIDATION_MSGS["field"]["empty_value"].format("workspace_id") in self.validator.errors[0] ) def test_validate_workspace_field_invalid_type(self): """Test _validate_workspace_field with invalid type.""" core = {"workspace_id": 123} result = self.validator._validate_workspace_field(core, "workspace_id") assert result is False assert len(self.validator.errors) == 1 assert "must be either a string or environment mapping" in self.validator.errors[0] def test_validate_environment_mapping_valid(self): """Test _validate_environment_mapping with valid mapping.""" field_value = {"dev": "dev-value", "prod": "prod-value"} result = self.validator._validate_environment_mapping(field_value, "test_field", str) assert result is True assert self.validator.errors == [] def test_validate_environment_mapping_empty(self): """Test _validate_environment_mapping with empty mapping.""" field_value = {} result = self.validator._validate_environment_mapping(field_value, "test_field", str) assert result is False assert len(self.validator.errors) == 1 assert "environment mapping cannot be empty" in self.validator.errors[0] def test_validate_environment_mapping_invalid_env_key(self): """Test _validate_environment_mapping with invalid environment key.""" field_value = {"": "value", "dev": "dev-value"} result = self.validator._validate_environment_mapping(field_value, "test_field", str) assert result is False assert len(self.validator.errors) == 1 assert "Environment key in 'test_field' must be a non-empty string" in self.validator.errors[0] def test_validate_environment_mapping_wrong_value_type(self): """Test _validate_environment_mapping with wrong value type.""" field_value = {"dev": 123, "prod": "prod-value"} result = self.validator._validate_environment_mapping(field_value, "test_field", str) assert result is False assert len(self.validator.errors) == 1 assert "must be a str, got int" in self.validator.errors[0] def test_validate_environment_mapping_empty_string_value(self): """Test _validate_environment_mapping with empty string value.""" field_value = {"dev": "", "prod": "prod-value"} result = self.validator._validate_environment_mapping(field_value, "test_field", str) assert result is False assert len(self.validator.errors) == 1 assert "value for environment 'dev' cannot be empty" in self.validator.errors[0] def test_validate_environment_mapping_empty_list_value(self): """Test _validate_environment_mapping with empty list value.""" field_value = {"dev": [], "prod": ["item1"]} result = self.validator._validate_environment_mapping(field_value, "test_field", list) assert result is False assert len(self.validator.errors) == 1 assert "value for environment 'dev' cannot be empty" in self.validator.errors[0] def test_validate_repository_directory_empty_string(self): """Test _validate_repository_directory with empty string value.""" core = {"repository_directory": " "} self.validator._validate_repository_directory(core) assert len(self.validator.errors) == 1 assert ( constants.CONFIG_VALIDATION_MSGS["field"]["empty_value"].format("repository_directory") in self.validator.errors[0] ) def test_validate_repository_directory_valid_string(self): """Test _validate_repository_directory with valid string.""" core = {"repository_directory": "/path/to/repo"} self.validator._validate_repository_directory(core) assert self.validator.errors == [] def test_validate_repository_directory_missing(self): """Test _validate_repository_directory with missing field.""" core = {} self.validator._validate_repository_directory(core) assert len(self.validator.errors) == 1 assert "must specify 'repository_directory'" in self.validator.errors[0] def test_validate_repository_directory_invalid_type(self): """Test _validate_repository_directory with invalid type.""" core = {"repository_directory": 123} self.validator._validate_repository_directory(core) assert len(self.validator.errors) == 1 assert "must be either a string or environment mapping" in self.validator.errors[0] def test_validate_repository_directory_valid_env_mapping(self): """Test _validate_repository_directory with valid environment mapping.""" core = {"repository_directory": {"dev": "/dev/path", "prod": "/prod/path"}} self.validator._validate_repository_directory(core) assert self.validator.errors == [] def test_validate_repository_directory_invalid_env_mapping(self): """Test _validate_repository_directory with invalid environment mapping.""" core = {"repository_directory": {"": "/dev/path"}} self.validator._validate_repository_directory(core) assert len(self.validator.errors) == 1 assert ( constants.CONFIG_VALIDATION_MSGS["environment"]["invalid_env_key"].format("repository_directory", "str") in self.validator.errors[0] ) def test_validate_item_types_valid_list(self): """Test _validate_item_types with valid item types.""" item_types = ["Notebook", "DataPipeline"] self.validator._validate_item_types(item_types) assert self.validator.errors == [] def test_validate_item_types_empty_list(self): """Test _validate_item_types with empty list.""" item_types = [] self.validator._validate_item_types(item_types) assert len(self.validator.errors) == 1 assert "'item_types_in_scope' cannot be empty" in self.validator.errors[0] def test_validate_item_types_invalid_type(self): """Test _validate_item_types with invalid item type.""" item_types = ["Notebook", 123, "DataPipeline"] self.validator._validate_item_types(item_types) assert len(self.validator.errors) == 1 assert "Item type must be a string, got int" in self.validator.errors[0] def test_validate_item_types_unknown_item_type(self): """Test _validate_item_types with unknown item type.""" item_types = ["Notebook", "UnknownType"] self.validator._validate_item_types(item_types) assert len(self.validator.errors) == 1 assert "Invalid item type 'UnknownType'" in self.validator.errors[0] assert "Available types:" in self.validator.errors[0] def test_validate_item_types_with_env_context(self): """Test _validate_item_types with environment context.""" item_types = ["UnknownType"] self.validator._validate_item_types(item_types, env_context="dev") assert len(self.validator.errors) == 1 assert "Invalid item type 'UnknownType' in environment 'dev'" in self.validator.errors[0] def test_validate_item_types_in_scope_invalid_env_mapping(self): """Test _validate_item_types_in_scope with invalid environment mapping.""" core = {"item_types_in_scope": {"": ["Notebook"]}} self.validator._validate_item_types_in_scope(core) assert len(self.validator.errors) == 1 assert ( constants.CONFIG_VALIDATION_MSGS["environment"]["invalid_env_key"].format("item_types_in_scope", "str") in self.validator.errors[0] ) def test_validate_regex_valid(self): """Test _validate_regex with valid regex.""" self.validator._validate_regex("^test.*", "test_section") assert self.validator.errors == [] def test_validate_regex_invalid(self): """Test _validate_regex with invalid regex.""" self.validator._validate_regex("[invalid", "test_section") assert len(self.validator.errors) == 1 assert "is not a valid regex pattern" in self.validator.errors[0] def test_validate_items_list_valid(self): """Test _validate_items_list with valid items.""" items_list = ["item1.Notebook", "item2.DataPipeline"] self.validator._validate_items_list(items_list, "test_context") assert self.validator.errors == [] def test_validate_items_list_invalid_type(self): """Test _validate_items_list with invalid item type.""" items_list = ["item1.Notebook", 123] self.validator._validate_items_list(items_list, "test_context") assert len(self.validator.errors) == 1 assert "'test_context[1]' must be a string" in self.validator.errors[0] def test_validate_items_list_empty_item(self): """Test _validate_items_list with empty item.""" items_list = ["item1.Notebook", ""] self.validator._validate_items_list(items_list, "test_context") assert len(self.validator.errors) == 1 assert ( constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_empty"].format("test_context", 1) in self.validator.errors[0] ) def test_validate_features_list_valid(self): """Test _validate_features_list with valid features.""" features_list = ["enable_shortcut_publish"] self.validator._validate_features_list(features_list, "test_context") assert self.validator.errors == [] def test_validate_features_list_invalid_type(self): """Test _validate_features_list with invalid feature type.""" features_list = ["enable_shortcut_publish", 123] self.validator._validate_features_list(features_list, "test_context") assert len(self.validator.errors) == 1 assert "'test_context[1]' must be a string" in self.validator.errors[0] def test_validate_features_list_empty_feature(self): """Test _validate_features_list with empty feature.""" features_list = ["enable_shortcut_publish", ""] self.validator._validate_features_list(features_list, "test_context") assert len(self.validator.errors) == 1 assert ( constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_empty"].format("test_context", 1) in self.validator.errors[0] ) @pytest.mark.parametrize("key_value", [123, ""]) def test_validate_constants_dict_invalid_or_empty_key(self, key_value): """Test _validate_constants_section with invalid or empty key.""" constants_dict = {key_value: "value"} self.validator._validate_constants_section(constants_dict) assert len(self.validator.errors) == 1 assert "Constant key in 'constants' must be a non-empty string" in self.validator.errors[0] def test_validate_constants_dict_unknown_constant(self): """Test _validate_constants_section with unknown constant.""" constants_dict = {"UNKNOWN_CONSTANT": "value"} self.validator._validate_constants_section(constants_dict) assert len(self.validator.errors) == 1 assert "Unknown constant 'UNKNOWN_CONSTANT'" in self.validator.errors[0] def test_validate_constants_dict_valid_various_types(self): """Test _validate_constants_section with valid constants of various types.""" constants_dict = { "DEFAULT_API_ROOT_URL": "https://api.fabric.microsoft.com", "ACCEPTED_ITEM_TYPES": ["Notebook", "DataPipeline"], } self.validator._validate_constants_section(constants_dict) assert self.validator.errors == [] def test_validate_constants_dict_url_constant_non_string_type(self): """Test _validate_constants_section rejects non-string URL constant.""" constants_dict = {"DEFAULT_API_ROOT_URL": 12345} self.validator._validate_constants_section(constants_dict) assert len(self.validator.errors) == 1 assert "'constants.DEFAULT_API_ROOT_URL' must be a string URL, got int" in self.validator.errors[0] def test_validate_constants_dict_url_constant_invalid_hostname(self): """Test _validate_constants_section rejects URL with invalid hostname.""" constants_dict = {"DEFAULT_API_ROOT_URL": "https://evil.example.com"} self.validator._validate_constants_section(constants_dict) assert len(self.validator.errors) == 1 assert "invalid hostname" in self.validator.errors[0].lower() def test_validate_constants_dict_url_constant_http_scheme(self): """Test _validate_constants_section rejects URL with HTTP scheme.""" constants_dict = {"DEFAULT_API_ROOT_URL": "http://api.fabric.microsoft.com"} self.validator._validate_constants_section(constants_dict) assert len(self.validator.errors) == 1 assert "must use HTTPS scheme" in self.validator.errors[0] def test_validate_constants_dict_url_constant_with_path(self): """Test _validate_constants_section rejects URL with path components.""" constants_dict = {"DEFAULT_API_ROOT_URL": "https://api.fabric.microsoft.com/v1"} self.validator._validate_constants_section(constants_dict) assert len(self.validator.errors) == 1 assert "without path components" in self.validator.errors[0] def test_validate_constants_dict_url_constant_valid_powerbi(self): """Test _validate_constants_section accepts valid PowerBI URL constant.""" constants_dict = {"FABRIC_API_ROOT_URL": "https://api.powerbi.com"} self.validator._validate_constants_section(constants_dict) assert self.validator.errors == [] def test_validate_constants_dict_non_url_constant_skips_url_validation(self): """Test _validate_constants_section skips URL validation for non-URL constants.""" constants_dict = {"ACCEPTED_ITEM_TYPES": ["Notebook", "DataPipeline"]} self.validator._validate_constants_section(constants_dict) assert self.validator.errors == [] def test_validate_item_types_in_scope_valid_list(self): """Test _validate_item_types_in_scope with valid list.""" core = {"item_types_in_scope": ["Notebook", "DataPipeline"]} self.validator._validate_item_types_in_scope(core) assert self.validator.errors == [] def test_validate_item_types_in_scope_empty_list(self): """Test _validate_item_types_in_scope with empty list.""" core = {"item_types_in_scope": []} self.validator._validate_item_types_in_scope(core) assert len(self.validator.errors) == 1 assert "'item_types_in_scope' cannot be empty if specified" in self.validator.errors[0] def test_validate_item_types_in_scope_environment_mapping(self): """Test _validate_item_types_in_scope with environment mapping.""" core = {"item_types_in_scope": {"dev": ["Notebook"], "prod": ["DataPipeline", "Notebook"]}} self.validator._validate_item_types_in_scope(core) assert self.validator.errors == [] def test_validate_item_types_in_scope_invalid_type(self): """Test _validate_item_types_in_scope with invalid type.""" core = {"item_types_in_scope": "invalid"} self.validator._validate_item_types_in_scope(core) assert len(self.validator.errors) == 1 assert "must be either a list or environment mapping dictionary" in self.validator.errors[0] def test_validate_item_types_in_scope_missing_field(self): """Test _validate_item_types_in_scope with missing field (should be okay).""" core = {"workspace_id": "12345678-1234-1234-1234-123456789abc"} self.validator._validate_item_types_in_scope(core) assert self.validator.errors == [] def test_resolve_repository_path_absolute_path(self, tmp_path): """Test _resolve_repository_path with absolute path.""" # Create actual directory repo_dir = tmp_path / "workspace" repo_dir.mkdir() self.validator.config = {"core": {"repository_directory": str(repo_dir)}} self.validator.config_path = tmp_path / "config.yaml" self.validator._resolve_repository_path() assert self.validator.errors == [] assert Path(self.validator.config["core"]["repository_directory"]) == repo_dir def test_resolve_repository_path_relative_path(self, tmp_path): """Test _resolve_repository_path with relative path.""" # Create actual directory structure config_dir = tmp_path / "configs" config_dir.mkdir() repo_dir = tmp_path / "workspace" repo_dir.mkdir() self.validator.config = {"core": {"repository_directory": "../workspace"}} self.validator.config_path = config_dir / "config.yaml" self.validator._resolve_repository_path() assert self.validator.errors == [] resolved_path = Path(self.validator.config["core"]["repository_directory"]) assert resolved_path.is_absolute() assert resolved_path.exists() def test_resolve_repository_path_nonexistent_directory(self, tmp_path): """Test _resolve_repository_path with nonexistent directory.""" self.validator.config = {"core": {"repository_directory": "nonexistent"}} self.validator.config_path = tmp_path / "config.yaml" self.validator._resolve_repository_path() assert len(self.validator.errors) == 1 assert "repository_directory not found at resolved path" in self.validator.errors[0] def test_resolve_repository_path_file_instead_of_directory(self, tmp_path): """Test _resolve_repository_path with file instead of directory.""" # Create a file instead of directory not_a_dir = tmp_path / "not_a_dir.txt" not_a_dir.write_text("content") self.validator.config = {"core": {"repository_directory": str(not_a_dir)}} self.validator.config_path = tmp_path / "config.yaml" self.validator._resolve_repository_path() assert len(self.validator.errors) == 1 assert "repository_directory path exists but is not a directory" in self.validator.errors[0] def test_resolve_repository_path_environment_mapping(self, tmp_path): """Test _resolve_repository_path with environment mapping.""" # Create actual directories dev_repo = tmp_path / "dev_workspace" dev_repo.mkdir() prod_repo = tmp_path / "prod_workspace" prod_repo.mkdir() self.validator.config = {"core": {"repository_directory": {"dev": str(dev_repo), "prod": str(prod_repo)}}} self.validator.config_path = tmp_path / "config.yaml" self.validator._resolve_repository_path() assert self.validator.errors == [] repo_dirs = self.validator.config["core"]["repository_directory"] assert Path(repo_dirs["dev"]).is_absolute() assert Path(repo_dirs["prod"]).is_absolute() def test_validate_parameter_field_valid_configurations(self): """Test parameter field validation with valid string and environment mapping.""" # Test valid string core_string = {"parameter": "parameter.yml"} self.validator._validate_parameter_field(core_string) assert self.validator.errors == [] # Reset for next test self.validator.errors = [] # Test valid environment mapping core_mapping = {"parameter": {"dev": "dev-parameter.yml", "prod": "prod-parameter.yml"}} self.validator._validate_parameter_field(core_mapping) assert self.validator.errors == [] def test_validate_parameter_field_invalid_configurations(self): """Test parameter field validation with invalid configurations.""" # Test empty string core_empty = {"parameter": ""} self.validator._validate_parameter_field(core_empty) assert len(self.validator.errors) == 1 assert constants.CONFIG_VALIDATION_MSGS["field"]["empty_value"].format("parameter") in self.validator.errors[0] # Reset for next test self.validator.errors = [] # Test invalid type core_invalid_type = {"parameter": 123} self.validator._validate_parameter_field(core_invalid_type) assert len(self.validator.errors) == 1 assert ( constants.CONFIG_VALIDATION_MSGS["field"]["string_or_dict"].format("parameter", "int") in self.validator.errors[0] ) def test_validate_parameter_field_invalid_env_mapping(self): """Test _validate_parameter_field with invalid environment mapping.""" core = {"parameter": {"": "param.yml"}} self.validator._validate_parameter_field(core) assert len(self.validator.errors) == 1 assert ( constants.CONFIG_VALIDATION_MSGS["environment"]["invalid_env_key"].format("parameter", "str") in self.validator.errors[0] ) def test_resolve_parameter_path_basic_functionality(self, tmp_path): """Test basic parameter path resolution functionality.""" # Create parameter file param_file = tmp_path / "parameter.yml" param_file.write_text("find_replace: []") self.validator.config = { "core": { "workspace_id": "12345678-1234-1234-1234-123456789abc", "repository_directory": "workspace", "parameter": "parameter.yml", } } self.validator.config_path = tmp_path / "config.yml" self.validator._resolve_parameter_path() assert self.validator.errors == [] resolved_path = Path(self.validator.config["core"]["parameter"]) assert resolved_path.is_absolute() assert resolved_path.exists() assert resolved_path.name == "parameter.yml" def test_resolve_path_field_directory_relative_path(self, tmp_path): """Test _resolve_path_field with relative directory path.""" # Create directory test_dir = tmp_path / "test_dir" test_dir.mkdir() self.validator.config = {"test_section": {"test_field": "test_dir"}} self.validator.config_path = tmp_path / "config.yml" self.validator._resolve_path_field("test_dir", "test_field", "test_section", "directory") assert self.validator.errors == [] resolved_path = Path(self.validator.config["test_section"]["test_field"]) assert resolved_path.is_absolute() assert resolved_path.exists() assert resolved_path.is_dir() def test_resolve_path_field_file_absolute_path(self, tmp_path): """Test _resolve_path_field with absolute file path.""" # Create file test_file = tmp_path / "test_file.txt" test_file.write_text("test content") self.validator.config = {"test_section": {"test_field": str(test_file)}} self.validator.config_path = tmp_path / "config.yml" self.validator._resolve_path_field(str(test_file), "test_field", "test_section", "file") assert self.validator.errors == [] resolved_path = Path(self.validator.config["test_section"]["test_field"]) assert resolved_path.is_absolute() assert resolved_path.exists() assert resolved_path.is_file() def test_resolve_path_field_git_repo_mismatch(self, tmp_path): """Test _resolve_path_field detects different git repositories.""" test_dir = tmp_path / "test_dir" test_dir.mkdir() self.validator.config = {"test_section": {"test_field": str(test_dir)}} self.validator.config_path = tmp_path / "config.yml" with patch( "fabric_cicd._common._config_validator._find_git_root", side_effect=[Path("/repo1"), Path("/repo2")], ): self.validator._resolve_path_field(str(test_dir), "test_field", "test_section", "directory") assert len(self.validator.errors) == 1 assert "same git repository" in self.validator.errors[0] def test_resolve_path_field_file_type_but_is_directory(self, tmp_path): """Test _resolve_path_field when expecting file but path is a directory.""" test_dir = tmp_path / "a_dir" test_dir.mkdir() self.validator.config = {"test_section": {"test_field": "a_dir"}} self.validator.config_path = tmp_path / "config.yml" self.validator._resolve_path_field("a_dir", "test_field", "test_section", "file") assert len(self.validator.errors) == 1 assert "test_field path exists but is not a file" in self.validator.errors[0] def test_resolve_path_field_os_error(self, tmp_path): """Test _resolve_path_field with OSError during path resolution.""" self.validator.config = {"test_section": {"test_field": "some_path"}} self.validator.config_path = tmp_path / "config.yml" with patch.object(Path, "resolve", side_effect=OSError("bad path")): self.validator._resolve_path_field("some_path", "test_field", "test_section", "directory") assert len(self.validator.errors) == 1 assert "Invalid test_field path" in self.validator.errors[0] assert "some_path" in self.validator.errors[0] assert "bad path" in self.validator.errors[0] def test_resolve_path_field_environment_mapping(self, tmp_path): """Test _resolve_path_field with environment mapping.""" self.validator.environment = "DEV" # Create directories for different environments dev_dir = tmp_path / "dev_dir" prod_dir = tmp_path / "prod_dir" dev_dir.mkdir() prod_dir.mkdir() field_value = {"DEV": "dev_dir", "PROD": "prod_dir"} self.validator.config = {"test_section": {"test_field": field_value}} self.validator.config_path = tmp_path / "config.yml" self.validator._resolve_path_field(field_value, "test_field", "test_section", "directory") assert self.validator.errors == [] # Only DEV environment should be resolved since that's the target environment resolved_path = Path(self.validator.config["test_section"]["test_field"]["DEV"]) assert resolved_path.is_absolute() assert resolved_path.exists() assert resolved_path.is_dir() # PROD should remain unchanged since it wasn't the target environment assert self.validator.config["test_section"]["test_field"]["PROD"] == "prod_dir" def test_resolve_path_field_environment_not_in_mapping(self, tmp_path): """Test _resolve_path_field skips gracefully when environment is not in mapping.""" self.validator.environment = "prod" # Target environment # Create directory only for 'dev' environment dev_dir = tmp_path / "dev_dir" dev_dir.mkdir() # Parameter mapping only has 'dev', not 'prod' field_value = {"dev": "dev_dir"} self.validator.config = {"test_section": {"test_field": field_value}} self.validator.config_path = tmp_path / "config.yml" # Should NOT raise KeyError - should skip gracefully self.validator._resolve_path_field(field_value, "test_field", "test_section", "directory") # No errors should be added (optional field behavior) assert self.validator.errors == [] # Config should remain unchanged since resolution was skipped assert self.validator.config["test_section"]["test_field"] == {"dev": "dev_dir"} def test_resolve_path_field_nonexistent_path(self, tmp_path): """Test _resolve_path_field with nonexistent path.""" self.validator.config = {"test_section": {"test_field": "nonexistent_dir"}} self.validator.config_path = tmp_path / "config.yml" self.validator._resolve_path_field("nonexistent_dir", "test_field", "test_section", "directory") assert len(self.validator.errors) == 1 assert "test_field not found at resolved path" in self.validator.errors[0] def test_resolve_path_field_wrong_type_file_vs_directory(self, tmp_path): """Test _resolve_path_field when path exists but is wrong type.""" # Create a file but try to resolve it as a directory test_file = tmp_path / "test_file.txt" test_file.write_text("test content") self.validator.config = {"test_section": {"test_field": "test_file.txt"}} self.validator.config_path = tmp_path / "config.yml" self.validator._resolve_path_field("test_file.txt", "test_field", "test_section", "directory") assert len(self.validator.errors) == 1 assert "test_field path exists but is not a directory" in self.validator.errors[0] def test_resolve_path_field_no_config_path(self): """Test _resolve_path_field when config_path is None (validation failed).""" self.validator.config_path = None # Simulate config validation failure self.validator.config = {"test_section": {"test_field": "test_dir"}} self.validator._resolve_path_field("test_dir", "test_field", "test_section", "directory") # Should skip resolution and not add any errors assert self.validator.errors == [] assert self.validator.config["test_section"]["test_field"] == "test_dir" # Unchanged def test_environment_exists_valid(self): """Test _validate_environment_exists with valid environment.""" self.validator.config = {"core": {"workspace_id": {"dev": "dev-id", "prod": "prod-id"}}} self.validator.environment = "dev" self.validator._validate_environment_exists() assert self.validator.errors == [] def test_environment_exists_missing_environment(self): """Test _validate_environment_exists with missing environment.""" self.validator.config = {"core": {"workspace_id": {"dev": "dev-id", "prod": "prod-id"}}} self.validator.environment = "test" self.validator._validate_environment_exists() assert len(self.validator.errors) == 1 assert "Environment 'test' not found in 'core.workspace_id' mappings" in self.validator.errors[0] def test_environment_exists_no_environment_with_mapping(self): """Test _validate_environment_exists with N/A environment but config has mappings.""" self.validator.config = {"core": {"workspace_id": {"dev": "dev-id", "prod": "prod-id"}}} self.validator.environment = "N/A" self.validator._validate_environment_exists() assert len(self.validator.errors) == 1 assert "Configuration contains environment mappings but no environment was provided" in self.validator.errors[0] def test_environment_exists_no_environment_no_mapping(self): """Test _validate_environment_exists with N/A environment and no mappings.""" self.validator.config = {"core": {"workspace_id": "single-id", "repository_directory": "/path/to/repo"}} self.validator.environment = "N/A" self.validator._validate_environment_exists() assert self.validator.errors == [] # Config Override Tests @pytest.mark.parametrize( ("section", "value", "expected_result", "expected_error_msg"), [ # Valid cases - expect True with no errors ("core", {"workspace_id": "new-id"}, True, None), ("features", ["feature1", "feature2"], True, None), ("features", {"dev": ["feature1"], "prod": ["feature2"]}, True, None), # Publish/Unpublish section cases ("publish", {"skip": False}, True, None), ("publish", {"exclude_regex": "^TEST.*"}, True, None), ("publish", {"items_to_include": ["item1.Notebook"]}, True, None), ("unpublish", {"skip": True}, True, None), ("unpublish", {"exclude_regex": "^OLD.*", "skip": False}, True, None), # Basic validation only - these pass but would be validated later ("features", ["feature1", 123], True, None), # Contains non-string ("constants", {123: "value"}, True, None), # Invalid key ("constants", {"UNKNOWN_CONSTANT": "value"}, True, None), # Unknown constant # Invalid cases - expect False with errors ( "invalid_section", {"field": "value"}, False, "Cannot override unsupported config section: 'invalid_section'", ), ("core", "invalid_type", False, "Override section 'core' must be a dict, got str"), ("core", {"invalid_setting": "value"}, False, "Cannot override unsupported setting 'core.invalid_setting'"), ("publish", "invalid_type", False, "Override section 'publish' must be a dict, got str"), ("unpublish", "invalid_type", False, "Override section 'unpublish' must be a dict, got str"), ( "publish", {"invalid_setting": "value"}, False, "Cannot override unsupported setting 'publish.invalid_setting'", ), ( "unpublish", {"invalid_setting": "value"}, False, "Cannot override unsupported setting 'unpublish.invalid_setting'", ), ], ) def test_valid_override_section(self, section, value, expected_result, expected_error_msg): """Test _valid_override_section with various inputs.""" # Reset errors before each test case self.validator.errors = [] result = self.validator._valid_override_section(section, value) assert result is expected_result if expected_error_msg: assert len(self.validator.errors) == 1 assert expected_error_msg in self.validator.errors[0] else: assert self.validator.errors == [] @pytest.mark.parametrize( ("section", "initial_config", "override_value", "expected_result"), [ # Features replacement ( "features", {"features": ["existing_feature"]}, ["new_feature1", "new_feature2"], {"features": ["new_feature1", "new_feature2"]}, ), # Constants merge ( "constants", {"constants": {"EXISTING_CONST": "existing_value"}}, {"NEW_CONST": "new_value"}, {"constants": {"NEW_CONST": "new_value"}}, ), # Constants create section ( "constants", {"core": {"workspace_id": "test"}}, {"NEW_CONST": "new_value"}, {"core": {"workspace_id": "test"}, "constants": {"NEW_CONST": "new_value"}}, ), # Publish section overrides ( "publish", {"publish": {"skip": True, "exclude_regex": "^OLD.*"}}, {"skip": False}, {"publish": {"skip": False, "exclude_regex": "^OLD.*"}}, ), # Unpublish section overrides ( "unpublish", {"unpublish": {"skip": False}}, {"skip": True, "exclude_regex": "^TEST.*"}, {"unpublish": {"skip": True, "exclude_regex": "^TEST.*"}}, ), # Create publish section with multiple settings ( "publish", {"core": {"workspace_id": "test-id"}}, {"skip": False, "exclude_regex": "^TEST.*", "items_to_include": ["item1.Notebook"]}, { "core": {"workspace_id": "test-id"}, "publish": {"skip": False, "exclude_regex": "^TEST.*", "items_to_include": ["item1.Notebook"]}, }, ), # Create unpublish section ( "unpublish", {"core": {"workspace_id": "test-id"}}, {"skip": True}, {"core": {"workspace_id": "test-id"}, "unpublish": {"skip": True}}, ), ], ) def test_merge_overrides_basic_sections(self, section, initial_config, override_value, expected_result): """Test _merge_overrides with various basic section operations.""" self.validator.config = initial_config.copy() self.validator.errors = [] self.validator._merge_overrides(section, override_value) assert self.validator.errors == [] assert self.validator.config == expected_result @pytest.mark.parametrize( ("initial_config", "override_value", "expected_config", "expected_error", "test_description"), [ # Direct value override ( {"core": {"workspace_id": "original-id", "repository_directory": "/original/path"}}, {"workspace_id": "new-id"}, {"core": {"workspace_id": "new-id", "repository_directory": "/original/path"}}, None, "Direct value override", ), # Environment-specific override ( {"core": {"workspace_id": {"dev": "original-dev-id", "prod": "prod-id"}}}, {"workspace_id": {"dev": "new-dev-id"}}, {"core": {"workspace_id": {"dev": "new-dev-id", "prod": "prod-id"}}}, None, "Environment-specific override", ), # Create environment mapping from simple value ( {"core": {"workspace_id": "simple-id"}}, {"workspace_id": {"dev": "new-dev-id"}}, {"core": {"workspace_id": {"dev": "new-dev-id"}}}, None, "Create environment mapping", ), # Add new optional field ( {"core": {"workspace_id": "test-id"}}, {"item_types_in_scope": ["Notebook"]}, {"core": {"workspace_id": "test-id", "item_types_in_scope": ["Notebook"]}}, None, "Add new optional field", ), # Create new publish section ( {"core": {"workspace_id": "test-id"}}, {"skip": False}, {"core": {"workspace_id": "test-id"}, "publish": {"skip": False}}, None, "Create new section", ), # Environment-specific publish section ( {"core": {"workspace_id": "test-id"}}, {"skip": {"dev": False}}, {"core": {"workspace_id": "test-id"}, "publish": {"skip": {"dev": False}}}, None, "Environment-specific publish setting", ), # Environment-specific unpublish section ( {"core": {"workspace_id": "test-id"}}, {"exclude_regex": {"dev": "^TEST_DEV.*"}}, {"core": {"workspace_id": "test-id"}, "unpublish": {"exclude_regex": {"dev": "^TEST_DEV.*"}}}, None, "Environment-specific unpublish setting", ), # Cannot create core section ( {"features": ["test"]}, {"workspace_id": "test-id"}, {"features": ["test"]}, # Unchanged "Cannot create 'core' section", "Prevent creating core section", ), # Cannot create required repository_directory ( {"core": {"workspace_id": "test-id"}}, {"repository_directory": "/new/path"}, {"core": {"workspace_id": "test-id"}}, # Unchanged "Cannot create required field 'core.repository_directory'", "Prevent creating required field", ), # Can override existing repository_directory ( {"core": {"workspace_id": "test-id", "repository_directory": "/original/path"}}, {"repository_directory": "/new/path"}, {"core": {"workspace_id": "test-id", "repository_directory": "/new/path"}}, None, "Allow overriding existing required field", ), ], ) def test_merge_overrides_core_section( self, initial_config, override_value, expected_config, expected_error, test_description ): """Test _merge_overrides with core section operations.""" # Set environment for environment mapping tests if "environment" in test_description.lower(): self.validator.environment = "dev" else: self.validator.environment = "N/A" self.validator.config = initial_config.copy() self.validator.errors = [] section = "publish" if "publish" in str(expected_config) else "core" section = "unpublish" if "unpublish" in str(expected_config) else section self.validator._merge_overrides(section, override_value) if expected_error: assert len(self.validator.errors) == 1 assert expected_error in self.validator.errors[0] else: assert self.validator.errors == [] assert self.validator.config == expected_config @pytest.mark.parametrize( ("initial_config", "override_value", "expected_result", "expected_error"), [ # Cannot create workspace_id without existing workspace ( {"core": {"repository_directory": "/path"}}, {"workspace_id": "new-id"}, {"core": {"repository_directory": "/path"}}, "Cannot create workspace identifier 'core.workspace_id'", ), # Cannot create workspace without existing workspace_id ( {"core": {"repository_directory": "/path"}}, {"workspace": "new-workspace"}, {"core": {"repository_directory": "/path"}}, "Cannot create workspace identifier 'core.workspace'", ), # Can create workspace_id when workspace already exists ( {"core": {"workspace": "existing-workspace", "repository_directory": "/path"}}, {"workspace_id": "new-id"}, { "core": { "workspace": "existing-workspace", "repository_directory": "/path", "workspace_id": "new-id", } }, None, ), # Can create workspace when workspace_id already exists ( {"core": {"workspace_id": "existing-id", "repository_directory": "/path"}}, {"workspace": "new-workspace"}, { "core": { "workspace_id": "existing-id", "repository_directory": "/path", "workspace": "new-workspace", } }, None, ), # Can override existing workspace identifiers ( { "core": { "workspace_id": "original-id", "workspace": "original-workspace", "repository_directory": "/path", } }, {"workspace_id": "new-id", "workspace": "new-workspace"}, {"core": {"workspace_id": "new-id", "workspace": "new-workspace", "repository_directory": "/path"}}, None, ), ], ) def test_merge_overrides_workspace_identifiers( self, initial_config, override_value, expected_result, expected_error ): """Test _merge_overrides with workspace identifier operations.""" self.validator.config = initial_config.copy() self.validator.errors = [] self.validator._merge_overrides("core", override_value) if expected_error: assert len(self.validator.errors) == 1 assert expected_error in self.validator.errors[0] else: assert self.validator.errors == [] assert self.validator.config == expected_result @pytest.mark.parametrize( ("initial_config", "config_override", "expected_config", "expected_error", "mock_side_effect"), [ # Successful override ( {"core": {"workspace_id": "original-id"}}, {"core": {"workspace_id": "new-id"}, "constants": {"DEFAULT_API_ROOT_URL": "https://api.test.com"}}, {"core": {"workspace_id": "new-id"}, "constants": {"DEFAULT_API_ROOT_URL": "https://api.test.com"}}, None, None, ), # Publish and unpublish sections ( {"core": {"workspace_id": "original-id"}}, { "publish": {"skip": False, "exclude_regex": "^TEST.*"}, "unpublish": {"skip": True, "items_to_include": ["item1.Notebook"]}, }, { "core": {"workspace_id": "original-id"}, "publish": {"skip": False, "exclude_regex": "^TEST.*"}, "unpublish": {"skip": True, "items_to_include": ["item1.Notebook"]}, }, None, None, ), # Empty publish section (should be rejected in real validation) ( {"core": {"workspace_id": "original-id"}}, {"publish": {}}, {"core": {"workspace_id": "original-id"}, "publish": {}}, None, # No error at override level, but would fail in section validation None, ), # Validation failure ( {"core": {"workspace_id": "original-id"}}, {"core": {"invalid_setting": "value"}}, {"core": {"workspace_id": "original-id"}}, # Config remains unchanged "Cannot override unsupported setting 'core.invalid_setting'", None, ), # Exception handling ( {"core": {"workspace_id": "original-id"}}, {"core": {"workspace_id": "new-id"}}, {"core": {"workspace_id": "original-id"}}, # Config remains unchanged "Failed to apply config override for section 'core': Test exception", Exception("Test exception"), ), # No overrides ( {"core": {"workspace_id": "original-id"}}, None, {"core": {"workspace_id": "original-id"}}, # Config remains unchanged None, None, ), ], ) def test_apply_and_validate_overrides( self, initial_config, config_override, expected_config, expected_error, mock_side_effect ): """Test _apply_and_validate_overrides with various scenarios.""" self.validator.config = initial_config.copy() self.validator.config_override = config_override self.validator.errors = [] if mock_side_effect: with patch.object(self.validator, "_merge_overrides", side_effect=mock_side_effect): self.validator._apply_and_validate_overrides() else: self.validator._apply_and_validate_overrides() if expected_error: assert len(self.validator.errors) == 1 assert expected_error in self.validator.errors[0] else: assert self.validator.errors == [] assert self.validator.config == expected_config def test_validate_config_file_with_overrides_integration(self, tmp_path): """Integration test for validate_config_file with config overrides.""" # Create a valid config file config_content = """ core: workspace_id: "12345678-1234-1234-1234-123456789abc" repository_directory: "workspace" constants: DEFAULT_API_ROOT_URL: "https://api.fabric.microsoft.com" """ config_file = tmp_path / "config.yaml" config_file.write_text(config_content) # Create workspace directory workspace_dir = tmp_path / "workspace" workspace_dir.mkdir() config_override = { "core": {"workspace_id": "87654321-4321-4321-4321-123456789abc"}, "constants": {"DEFAULT_API_ROOT_URL": "https://api.powerbi.com"}, } result = self.validator.validate_config_file(str(config_file), "DEV", config_override) assert result["core"]["workspace_id"] == "87654321-4321-4321-4321-123456789abc" assert result["constants"]["DEFAULT_API_ROOT_URL"] == "https://api.powerbi.com" def test_validate_config_file_with_invalid_overrides_integration(self, tmp_path): """Integration test for validate_config_file with invalid overrides.""" # Create a valid config file config_content = """ core: workspace_id: "12345678-1234-1234-1234-123456789abc" repository_directory: "workspace" """ config_file = tmp_path / "config.yaml" config_file.write_text(config_content) # Create workspace directory workspace_dir = tmp_path / "workspace" workspace_dir.mkdir() config_override = {"core": {"invalid_field": "value"}} with pytest.raises(ConfigValidationError) as exc_info: self.validator.validate_config_file(str(config_file), "DEV", config_override) assert "Cannot override unsupported setting 'core.invalid_field'" in str(exc_info.value) # Tests for utility functions class TestConfigValidatorUtilityFunctions: """Tests for standalone utility functions in the config validator module.""" def test_find_git_root_with_git_repo(self, tmp_path): """Test _find_git_root when path is in a git repository.""" from fabric_cicd._common._config_validator import _find_git_root # Create a fake git repo structure git_dir = tmp_path / ".git" git_dir.mkdir() # Test from root result = _find_git_root(tmp_path) assert result == tmp_path # Test from subdirectory sub_dir = tmp_path / "subdir" / "deep" sub_dir.mkdir(parents=True) result = _find_git_root(sub_dir) assert result == tmp_path def test_find_git_root_no_git_repo(self, tmp_path): """Test _find_git_root when path is not in a git repository.""" from fabric_cicd._common._config_validator import _find_git_root result = _find_git_root(tmp_path) assert result is None def test_validate_guid_format_valid(self): """Test _validate_guid_format with valid GUIDs.""" from fabric_cicd._common._config_validator import _validate_guid_format valid_guids = [ "12345678-1234-1234-1234-123456789abc", "ABCDEF12-3456-7890-ABCD-EF1234567890", "00000000-0000-0000-0000-000000000000", ] for guid in valid_guids: assert _validate_guid_format(guid) is True def test_validate_guid_format_invalid(self): """Test _validate_guid_format with invalid GUIDs.""" from fabric_cicd._common._config_validator import _validate_guid_format invalid_guids = [ "invalid-guid", "12345678-1234-1234-1234", # too short "12345678-1234-1234-1234-123456789abcd", # too long "12345678_1234_1234_1234_123456789abc", # wrong separators "", "not-a-guid-at-all", ] for guid in invalid_guids: assert _validate_guid_format(guid) is False def test_get_config_fields_complete_config(self): """Test _get_config_fields with complete configuration.""" from fabric_cicd._common._config_validator import _get_config_fields config = { "core": { "workspace_id": "test-id", "workspace": "test-workspace", "repository_directory": "/path", "item_types_in_scope": ["Notebook"], "parameter": "param.yml", }, "publish": { "exclude_regex": ".*_test", "folder_exclude_regex": "^/temp", "folder_path_to_include": ["/subfolder"], "shortcut_exclude_regex": "^shortcut_temp/", "items_to_include": ["item1"], "skip": False, }, "unpublish": {"exclude_regex": ".*_old", "items_to_include": ["item2"], "skip": True}, "features": ["feature1"], "constants": {"KEY": "value"}, } fields = _get_config_fields(config) # Should return all fields from all sections assert len(fields) == 16 # Updated count with folder_exclude_regex and shortcut_exclude_regex # Check some specific fields field_names = [field[1] for field in fields] assert "workspace_id" in field_names assert "repository_directory" in field_names assert "parameter" in field_names assert "folder_exclude_regex" in field_names assert "shortcut_exclude_regex" in field_names assert "features" in field_names assert "constants" in field_names # Check required vs optional flags for _section, field_name, _display_name, is_required, warn_if_missing in fields: if field_name in ["workspace_id", "workspace", "repository_directory"]: assert is_required is True, f"{field_name} should be required" else: assert is_required is False, f"{field_name} should be optional" if field_name in ["item_types_in_scope", "parameter"]: assert warn_if_missing is True, f"{field_name} should warn if missing" class TestConfigValidatorIntegration: """Integration tests for ConfigValidator.validate_config_file method.""" def test_validate_config_file_complete_success(self, tmp_path): """Test validate_config_file with complete valid configuration.""" # Create actual directory structure repo_dir = tmp_path / "workspace" repo_dir.mkdir() config_data = { "core": { "workspace_id": {"dev": "8b6e2c7a-4c1f-4e3a-9b2e-7d8f2e1a6c3b"}, "repository_directory": "workspace", "item_types_in_scope": ["Notebook", "DataPipeline"], }, "publish": {"exclude_regex": "^DONT_DEPLOY.*", "skip": {"dev": False}}, } config_file = tmp_path / "config.yaml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) validator = ConfigValidator() result = validator.validate_config_file(str(config_file), "dev") assert result is not None assert "core" in result assert "publish" in result # Path should be resolved to absolute assert Path(result["core"]["repository_directory"]).is_absolute() def test_validate_config_file_accumulates_errors(self, tmp_path): """Test validate_config_file accumulates multiple errors.""" config_data = { "core": { "workspace_id": 123, # Invalid type "item_types_in_scope": ["InvalidType"], # Invalid item type } # Missing repository_directory } config_file = tmp_path / "config.yaml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) validator = ConfigValidator() with pytest.raises(ConfigValidationError) as exc_info: validator.validate_config_file(str(config_file), "dev") # Should have multiple errors assert len(exc_info.value.validation_errors) >= 3 error_messages = " ".join(exc_info.value.validation_errors) assert "must be either a string or environment mapping" in error_messages assert "must specify 'repository_directory'" in error_messages assert "Invalid item type 'InvalidType'" in error_messages def test_validate_config_file_stops_at_yaml_parse_error(self, tmp_path): """Test validate_config_file stops at YAML parse error.""" config_file = tmp_path / "config.yaml" config_file.write_text("invalid: yaml: [") validator = ConfigValidator() with pytest.raises(ConfigValidationError) as exc_info: validator.validate_config_file(str(config_file), "dev") assert len(exc_info.value.validation_errors) == 1 assert "Invalid YAML syntax:" in exc_info.value.validation_errors[0] def test_validate_config_file_catches_guid_and_constants_errors(self, tmp_path): """Test validate_config_file catches GUID format and unknown constants errors.""" config_data = { "core": { "workspace_id": { "dev": "invalid-guid-format", # Invalid GUID "prod": "8b6e2c7a-4c1f-4e3a-9b2e-7d8f2e1a6c3b", # Valid GUID }, "repository_directory": "workspace", "item_types_in_scope": ["Notebook"], }, "constants": { "UNKNOWN_CONSTANT": "some_value", # Unknown constant "DEFAULT_API_ROOT_URL": "https://api.fabric.microsoft.com", # Valid URL constant }, } config_file = tmp_path / "config.yaml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) validator = ConfigValidator() with pytest.raises(ConfigValidationError) as exc_info: validator.validate_config_file(str(config_file), "dev") # Should catch both GUID format error and unknown constant error assert len(exc_info.value.validation_errors) >= 2 error_messages = " ".join(exc_info.value.validation_errors) assert "must be a valid GUID format" in error_messages assert "Unknown constant 'UNKNOWN_CONSTANT'" in error_messages class TestConfigSectionValidation: """Tests for section validation - required vs optional sections.""" def setup_method(self): """Set up for each test method.""" self.validator = ConfigValidator() def test_validate_config_sections_missing_core(self): """Test _validate_config_sections with missing core section.""" self.validator.config = {"publish": {"skip": False}, "unpublish": {"skip": True}} self.validator._validate_config_sections() assert len(self.validator.errors) == 1 assert "Configuration must contain a 'core' section" in self.validator.errors[0] def test_validate_config_sections_core_not_dict(self): """Test _validate_config_sections with core section not being a dictionary.""" self.validator.config = {"core": "not a dict"} self.validator._validate_config_sections() assert len(self.validator.errors) == 1 assert "Configuration must contain a 'core' section" in self.validator.errors[0] def test_validate_config_sections_core_only(self): """Test _validate_config_sections with only required core section.""" self.validator.config = { "core": {"workspace_id": "8b6e2c7a-4c1f-4e3a-9b2e-7d8f2e1a6c3b", "repository_directory": "/path/to/repo"} } self.validator._validate_config_sections() assert self.validator.errors == [] def test_validate_config_sections_with_optional_sections(self): """Test _validate_config_sections with optional sections present.""" self.validator.config = { "core": {"workspace_id": "8b6e2c7a-4c1f-4e3a-9b2e-7d8f2e1a6c3b", "repository_directory": "/path/to/repo"}, "publish": {"skip": False}, "unpublish": {"skip": True}, "features": ["enable_shortcut_publish"], "constants": {"DEFAULT_API_ROOT_URL": "https://api.example.com"}, } with patch.object(constants, "DEFAULT_API_ROOT_URL", "original_value"): self.validator._validate_config_sections() assert self.validator.errors == ["'constants.DEFAULT_API_ROOT_URL' has invalid hostname: api.example.com"] def test_validate_config_sections_missing_workspace_identifier(self): """Test _validate_config_sections with missing workspace identifier.""" self.validator.config = {"core": {"repository_directory": "/path/to/repo"}} self.validator._validate_config_sections() assert len(self.validator.errors) == 1 assert "Configuration must specify either 'workspace_id' or 'workspace'" in self.validator.errors[0] class TestOperationSectionValidation: """Tests for publish/unpublish operation section validation.""" def setup_method(self): """Set up for each test method.""" self.validator = ConfigValidator() def test_validate_operation_section_valid_basic(self): """Test _validate_operation_section with valid basic configuration.""" section = { "exclude_regex": "^TEST.*", "items_to_include": ["item1.Notebook", "item2.DataPipeline"], "skip": False, } self.validator._validate_operation_section(section, "publish") assert self.validator.errors == [] def test_validate_operation_section_not_dict(self): """Test _validate_operation_section with non-dictionary section.""" section = "not a dict" self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 assert "'publish' section must be a dictionary" in self.validator.errors[0] # --- Parametrized regex field tests --- @pytest.mark.parametrize("regex_field", ["exclude_regex", "folder_exclude_regex", "shortcut_exclude_regex"]) def test_validate_operation_section_valid_regex_field(self, regex_field): """Test _validate_operation_section with valid regex field.""" section = {regex_field: "^DONT_DEPLOY.*"} self.validator._validate_operation_section(section, "publish") assert self.validator.errors == [] @pytest.mark.parametrize("regex_field", ["exclude_regex", "folder_exclude_regex", "shortcut_exclude_regex"]) def test_validate_operation_section_invalid_regex_field(self, regex_field): """Test _validate_operation_section with invalid regex field.""" section = {regex_field: "[invalid"} self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 assert "is not a valid regex pattern" in self.validator.errors[0] @pytest.mark.parametrize( ("regex_field", "empty_value"), [ ("exclude_regex", ""), ("folder_exclude_regex", " "), ("shortcut_exclude_regex", " "), ], ) def test_validate_operation_section_empty_regex_field(self, regex_field, empty_value): """Test _validate_operation_section with empty regex field.""" section = {regex_field: empty_value} self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 assert "empty" in self.validator.errors[0].lower() @pytest.mark.parametrize("regex_field", ["exclude_regex", "folder_exclude_regex", "shortcut_exclude_regex"]) def test_validate_operation_section_regex_field_invalid_type(self, regex_field): """Test _validate_operation_section with regex field invalid type.""" section = {regex_field: 123} self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 assert "must be either a string or environment mapping" in self.validator.errors[0] @pytest.mark.parametrize("regex_field", ["exclude_regex", "folder_exclude_regex", "shortcut_exclude_regex"]) def test_validate_operation_section_regex_field_environment_mapping(self, regex_field): """Test _validate_operation_section with regex field environment mapping.""" section = {regex_field: {"dev": "^DEV_.*", "prod": "^PROD_.*"}} self.validator._validate_operation_section(section, "publish") assert self.validator.errors == [] @pytest.mark.parametrize("regex_field", ["exclude_regex", "folder_exclude_regex", "shortcut_exclude_regex"]) def test_validate_operation_section_regex_field_invalid_env_mapping(self, regex_field): """Test regex field dict with invalid environment key causes early return.""" section = {regex_field: {123: "^pattern"}} self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 assert ( constants.CONFIG_VALIDATION_MSGS["environment"]["invalid_env_key"].format(f"publish.{regex_field}", "int") in self.validator.errors[0] ) @pytest.mark.parametrize("regex_field", ["folder_exclude_regex", "shortcut_exclude_regex"]) def test_validate_operation_section_regex_field_rejected_in_unpublish(self, regex_field): """Test publish-only regex fields are rejected in unpublish section.""" section = {regex_field: "^temp"} self.validator._validate_operation_section(section, "unpublish") assert len(self.validator.errors) == 1 assert regex_field in self.validator.errors[0] assert "unpublish" in self.validator.errors[0] # --- items_to_include tests --- def test_validate_operation_section_empty_items_to_include(self): """Test _validate_operation_section with empty items_to_include list.""" section = {"items_to_include": []} self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 assert ( constants.CONFIG_VALIDATION_MSGS["operation"]["empty_list"].format("publish.items_to_include") in self.validator.errors[0] ) def test_validate_operation_section_items_to_include_valid_env_mapping(self): """Test _validate_operation_section with valid items_to_include environment mapping.""" section = {"items_to_include": {"dev": ["item1.Notebook"], "prod": ["item2.DataPipeline"]}} self.validator._validate_operation_section(section, "publish") assert self.validator.errors == [] def test_validate_operation_section_items_to_include_invalid_env_mapping(self): """Test items_to_include dict with invalid environment key causes early return.""" section = {"items_to_include": {123: ["item1"]}} self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 assert ( constants.CONFIG_VALIDATION_MSGS["environment"]["invalid_env_key"].format("publish.items_to_include", "int") in self.validator.errors[0] ) def test_validate_operation_section_items_to_include_invalid_type(self): """Test _validate_operation_section with items_to_include invalid type.""" section = {"items_to_include": "not a list or dict"} self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 assert ( constants.CONFIG_VALIDATION_MSGS["field"]["list_or_dict"].format("publish.items_to_include", "str") in self.validator.errors[0] ) # --- skip tests --- def test_validate_operation_section_skip_boolean(self): """Test _validate_operation_section with skip as boolean.""" section = {"skip": True} self.validator._validate_operation_section(section, "unpublish") assert self.validator.errors == [] def test_validate_operation_section_skip_environment_mapping(self): """Test _validate_operation_section with skip environment mapping.""" section = {"skip": {"dev": True, "test": False, "prod": False}} self.validator._validate_operation_section(section, "unpublish") assert self.validator.errors == [] def test_validate_operation_section_skip_invalid_type(self): """Test _validate_operation_section with skip invalid type.""" section = {"skip": "not a boolean"} self.validator._validate_operation_section(section, "unpublish") assert len(self.validator.errors) == 1 modified_msg = ( constants.CONFIG_VALIDATION_MSGS["field"]["string_or_dict"] .format("unpublish.skip", "str") .replace("a string", "a boolean") ) assert modified_msg in self.validator.errors[0] def test_validate_operation_section_skip_invalid_env_mapping(self): """Test skip dict with invalid environment key causes early return.""" section = {"skip": {123: True}} self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 assert ( constants.CONFIG_VALIDATION_MSGS["environment"]["invalid_env_key"].format("publish.skip", "int") in self.validator.errors[0] ) # --- folder_path_to_include tests --- def test_validate_operation_section_folder_path_to_include_valid_list(self): """Test _validate_operation_section with valid folder_path_to_include list.""" section = {"folder_path_to_include": ["/FolderA", "/FolderB"]} self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 0 def test_validate_operation_section_folder_path_to_include_valid_env_mapping(self): """Test _validate_operation_section with valid folder_path_to_include environment mapping.""" section = {"folder_path_to_include": {"dev": ["/FolderA"], "prod": ["/FolderA", "/FolderB"]}} self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 0 def test_validate_operation_section_folder_path_to_include_empty_list(self): """Test _validate_operation_section with empty folder_path_to_include list.""" section = {"folder_path_to_include": []} self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 assert ( constants.CONFIG_VALIDATION_MSGS["operation"]["empty_list"].format("publish.folder_path_to_include") in self.validator.errors[0] ) def test_validate_operation_section_folder_path_to_include_invalid_type(self): """Test _validate_operation_section with folder_path_to_include invalid type.""" section = {"folder_path_to_include": "not a list or dict"} self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 assert ( constants.CONFIG_VALIDATION_MSGS["field"]["list_or_dict"].format("publish.folder_path_to_include", "str") in self.validator.errors[0] ) def test_validate_operation_section_folder_path_to_include_unsupported_in_unpublish(self): """Test _validate_operation_section with folder_path_to_include in unpublish section.""" section = {"folder_path_to_include": ["/FolderA"]} self.validator._validate_operation_section(section, "unpublish") assert len(self.validator.errors) == 1 assert ( constants.CONFIG_VALIDATION_MSGS["operation"]["unsupported_field"].format( "folder_path_to_include", "unpublish" ) in self.validator.errors[0] ) def test_validate_operation_section_folder_path_to_include_entry_not_string(self): """Test _validate_operation_section with non-string entry in folder_path_to_include.""" section = {"folder_path_to_include": ["/FolderA", 123, "/FolderB"]} self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 assert ( constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_type"].format( "publish.folder_path_to_include", 1, "int" ) in self.validator.errors[0] ) def test_validate_operation_section_folder_path_to_include_empty_string_entry(self): """Test _validate_operation_section with empty string entry in folder_path_to_include.""" section = {"folder_path_to_include": ["/FolderA", "", "/FolderB"]} self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 assert ( constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_empty"].format( "publish.folder_path_to_include", 1 ) in self.validator.errors[0] ) def test_validate_operation_section_folder_path_to_include_missing_prefix(self): """Test _validate_operation_section with folder entry missing leading slash.""" section = {"folder_path_to_include": ["/FolderA", "FolderB"]} self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 assert ( constants.CONFIG_VALIDATION_MSGS["operation"]["folders_list_prefix"].format( "publish.folder_path_to_include", 1, "FolderB" ) in self.validator.errors[0] ) def test_validate_operation_section_folder_path_to_include_env_mapping_empty_list(self): """Test _validate_operation_section with empty list in environment mapping for folder_path_to_include.""" section = {"folder_path_to_include": {"dev": ["/FolderA"], "prod": []}} self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 assert ( constants.CONFIG_VALIDATION_MSGS["environment"]["empty_env_value"].format( "publish.folder_path_to_include", "prod" ) in self.validator.errors[0] ) def test_validate_operation_section_folder_path_to_include_env_mapping_invalid_entry(self): """Test _validate_operation_section with invalid entry in environment mapping for folder_path_to_include.""" section = {"folder_path_to_include": {"dev": ["/FolderA", "NoSlash"]}} self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 assert ( constants.CONFIG_VALIDATION_MSGS["operation"]["folders_list_prefix"].format( "publish.folder_path_to_include.dev", 1, "NoSlash" ) in self.validator.errors[0] ) def test_validate_operation_section_folder_path_to_include_nested_path(self): """Test _validate_operation_section with nested folder paths in folder_path_to_include.""" section = {"folder_path_to_include": ["/FolderA/SubFolder", "/FolderB/Sub1/Sub2"]} self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 0 def test_validate_operation_section_folder_path_to_include_whitespace_entry(self): """Test _validate_operation_section with whitespace-only entry in folder_path_to_include.""" section = {"folder_path_to_include": ["/FolderA", " "]} self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 assert ( constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_empty"].format( "publish.folder_path_to_include", 1 ) in self.validator.errors[0] ) # --- Mutual exclusivity tests --- def test_validate_operation_section_mutually_exclusive_both_direct_values(self): """Test that both folder_exclude_regex and folder_path_to_include as direct values raises error.""" section = {"folder_exclude_regex": "^/legacy", "folder_path_to_include": ["/subfolder"]} self.validator._validate_operation_section(section, "publish") error_messages = " ".join(self.validator.errors) assert ( constants.CONFIG_VALIDATION_MSGS["operation"]["mutually_exclusive"].format( "publish.folder_exclude_regex", "publish.folder_path_to_include" ) in error_messages ) def test_validate_operation_section_mutually_exclusive_both_env_mapped_overlapping(self): """Test that both fields with overlapping environment mappings raises error.""" self.validator.environment = "dev" section = { "folder_exclude_regex": {"dev": "^/legacy"}, "folder_path_to_include": {"dev": ["/subfolder"]}, } self.validator._validate_operation_section(section, "publish") error_messages = " ".join(self.validator.errors) assert ( constants.CONFIG_VALIDATION_MSGS["operation"]["mutually_exclusive_env"].format( "publish.folder_exclude_regex", "publish.folder_path_to_include", ["dev"] ) in error_messages ) def test_validate_operation_section_mutually_exclusive_both_env_mapped_no_overlap(self): """Test that both fields with non-overlapping environment mappings is valid.""" self.validator.environment = "dev" section = { "folder_exclude_regex": {"dev": "^/legacy"}, "folder_path_to_include": {"prod": ["/subfolder"]}, } self.validator._validate_operation_section(section, "publish") # Filter errors to only mutual exclusivity errors mutual_errors = [ e for e in self.validator.errors if "mutually exclusive" in e.lower() or "Cannot specify both" in e ] assert len(mutual_errors) == 0 def test_validate_operation_section_mutually_exclusive_direct_and_env_mapped_conflict(self): """Test that direct value + env-mapped value conflicts when target env matches.""" self.validator.environment = "dev" section = { "folder_exclude_regex": "^/legacy", # direct, applies to all "folder_path_to_include": {"dev": ["/subfolder"]}, # env-mapped, applies to dev } self.validator._validate_operation_section(section, "publish") error_messages = " ".join(self.validator.errors) assert ( constants.CONFIG_VALIDATION_MSGS["operation"]["mutually_exclusive_env"].format( "publish.folder_exclude_regex", "publish.folder_path_to_include", ["dev"] ) in error_messages ) def test_validate_operation_section_mutually_exclusive_direct_and_env_mapped_no_conflict(self): """Test that direct value + env-mapped value is valid when target env doesn't match.""" self.validator.environment = "prod" section = { "folder_exclude_regex": "^/legacy", # direct, applies to all "folder_path_to_include": {"dev": ["/subfolder"]}, # env-mapped, only dev } self.validator._validate_operation_section(section, "publish") # The direct value applies to prod, but folder_path_to_include doesn't apply to prod mutual_errors = [ e for e in self.validator.errors if "mutually exclusive" in e.lower() or "Cannot specify both" in e ] assert len(mutual_errors) == 0 def test_validate_operation_section_mutually_exclusive_env_mapped_and_direct_conflict(self): """Test that env-mapped value + direct value conflicts when target env matches.""" self.validator.environment = "dev" section = { "folder_exclude_regex": {"dev": "^/legacy"}, # env-mapped, applies to dev "folder_path_to_include": ["/subfolder"], # direct, applies to all } self.validator._validate_operation_section(section, "publish") error_messages = " ".join(self.validator.errors) assert ( constants.CONFIG_VALIDATION_MSGS["operation"]["mutually_exclusive_env"].format( "publish.folder_exclude_regex", "publish.folder_path_to_include", ["dev"] ) in error_messages ) def test_validate_operation_section_mutually_exclusive_only_one_field_present(self): """Test that having only one of the mutually exclusive fields is valid.""" section = {"folder_exclude_regex": "^/legacy"} self.validator._validate_operation_section(section, "publish") mutual_errors = [ e for e in self.validator.errors if "mutually exclusive" in e.lower() or "Cannot specify both" in e ] assert len(mutual_errors) == 0 class TestFeaturesSectionValidation: """Tests for features section validation.""" def setup_method(self): """Set up for each test method.""" self.validator = ConfigValidator() def test_validate_features_section_list(self): """Test _validate_features_section with list of features.""" features = ["enable_shortcut_publish", "feature2"] self.validator._validate_features_section(features) assert self.validator.errors == [] def test_validate_features_section_empty_list(self): """Test _validate_features_section with empty list.""" features = [] self.validator._validate_features_section(features) assert len(self.validator.errors) == 1 assert "'features' section cannot be empty if specified" in self.validator.errors[0] def test_validate_features_section_environment_mapping(self): """Test _validate_features_section with environment mapping.""" features = {"dev": ["enable_shortcut_publish"], "prod": ["feature2", "feature3"]} self.validator._validate_features_section(features) assert self.validator.errors == [] def test_validate_features_section_invalid_type(self): """Test _validate_features_section with invalid type.""" features = "not a list or dict" self.validator._validate_features_section(features) assert len(self.validator.errors) == 1 assert constants.CONFIG_VALIDATION_MSGS["operation"]["features_type"].format("str") in self.validator.errors[0] def test_validate_features_section_env_mapping_empty_list_for_env(self): """Test _validate_features_section with empty feature list for one environment.""" features = {"dev": ["feature1"], "prod": []} self.validator._validate_features_section(features) assert len(self.validator.errors) == 1 assert ( constants.CONFIG_VALIDATION_MSGS["environment"]["empty_env_value"].format("features", "prod") in self.validator.errors[0] ) class TestConstantsSectionValidation: """Tests for constants section validation.""" def setup_method(self): """Set up for each test method.""" self.validator = ConfigValidator() def test_validate_constants_section_dict(self): """Test _validate_constants_section with valid constants dictionary.""" constants_section = {"DEFAULT_API_ROOT_URL": "https://api.fabric.microsoft.com"} with patch.object(constants, "DEFAULT_API_ROOT_URL", "original_value"): self.validator._validate_constants_section(constants_section) assert self.validator.errors == [] def test_validate_constants_section_not_dict(self): """Test _validate_constants_section with non-dictionary.""" constants_section = "not a dict" self.validator._validate_constants_section(constants_section) assert len(self.validator.errors) == 1 assert "'constants' section must be a dictionary" in self.validator.errors[0] def test_validate_constants_section_environment_mapping(self): """Test _validate_constants_section with per-key environment mapping.""" constants_section = { "DEFAULT_API_ROOT_URL": { "dev": "https://api.fabric.microsoft.com", "prod": "https://api.powerbi.com", }, } self.validator._validate_constants_section(constants_section) assert self.validator.errors == [] def test_validate_constants_section_rejects_env_at_top_format(self): """Test _validate_constants_section rejects invalid env-at-top format.""" constants_section = { "dev": {"DEFAULT_API_ROOT_URL": "https://api.fabric.microsoft.com"}, "prod": {"DEFAULT_API_ROOT_URL": "https://api.powerbi.com"}, } self.validator._validate_constants_section(constants_section) assert len(self.validator.errors) == 2 assert "Unknown constant 'dev'" in self.validator.errors[0] assert "Unknown constant 'prod'" in self.validator.errors[1] def test_validate_constants_section_env_mapping_invalid_env_key(self): """Test _validate_constants_section with invalid env key in per-key mapping.""" constants_section = { "DEFAULT_API_ROOT_URL": {123: "https://api.fabric.microsoft.com"}, } self.validator._validate_constants_section(constants_section) assert len(self.validator.errors) == 1 assert ( constants.CONFIG_VALIDATION_MSGS["environment"]["invalid_env_key"].format( "constants.DEFAULT_API_ROOT_URL", "int" ) in self.validator.errors[0] ) assert "DEFAULT_API_ROOT_URL" in self.validator.errors[0] def test_validate_constants_section_env_mapping_url_validation(self): """Test _validate_constants_section validates URL in per-key env mapping.""" constants_section = { "DEFAULT_API_ROOT_URL": { "dev": "http://api.fabric.microsoft.com", # HTTP, not HTTPS }, } self.validator._validate_constants_section(constants_section) assert len(self.validator.errors) == 1 assert "must use HTTPS scheme" in self.validator.errors[0] assert "DEFAULT_API_ROOT_URL" in self.validator.errors[0] def test_validate_constants_section_env_mapping_non_string_url(self): """Test _validate_constants_section rejects non-string URL in per-key env mapping.""" constants_section = { "DEFAULT_API_ROOT_URL": {"dev": 12345}, } self.validator._validate_constants_section(constants_section) assert len(self.validator.errors) == 1 assert "must be a string URL" in self.validator.errors[0] assert "DEFAULT_API_ROOT_URL" in self.validator.errors[0] assert "int" in self.validator.errors[0] class TestEnvironmentMismatchValidation: """Tests for environment mismatch scenarios.""" def setup_method(self): """Set up for each test method.""" self.validator = ConfigValidator() def test_environment_mismatch_in_workspace_id(self): """Test environment exists validation with mismatch in workspace_id.""" self.validator.config = { "core": {"workspace_id": {"dev": "dev-id", "prod": "prod-id"}, "repository_directory": "/path/to/repo"} } self.validator.environment = "staging" # Not in the mapping self.validator._validate_environment_exists() assert len(self.validator.errors) == 1 assert "Environment 'staging' not found in 'core.workspace_id' mappings" in self.validator.errors[0] assert "Available: ['dev', 'prod']" in self.validator.errors[0] def test_environment_mismatch_in_multiple_fields(self): """Test environment exists validation with mismatches in multiple fields.""" self.validator.config = { "core": { "workspace_id": {"dev": "dev-id", "prod": "prod-id"}, "repository_directory": {"dev": "/dev/path", "prod": "/prod/path"}, "item_types_in_scope": {"dev": ["Notebook"], "prod": ["DataPipeline"]}, }, "publish": {"skip": {"dev": True, "prod": False}}, } self.validator.environment = "test" # Not in any mapping self.validator._validate_environment_exists() # Only required fields (workspace_id, repository_directory) cause errors # Optional fields (item_types_in_scope, skip) only log warnings/debug assert len(self.validator.errors) == 2 error_text = " ".join(self.validator.errors) assert "workspace_id" in error_text assert "repository_directory" in error_text def test_environment_mapping_vs_basic_values_mixed(self): """Test configuration with both environment mappings and basic values.""" self.validator.config = { "core": { "workspace_id": {"dev": "dev-id", "prod": "prod-id"}, # Environment mapping "repository_directory": "/single/path", # Basic value "item_types_in_scope": ["Notebook", "DataPipeline"], # Basic value }, "publish": { "skip": True # Basic boolean }, "unpublish": { "skip": {"dev": False, "prod": True} # Environment mapping }, } self.validator.environment = "dev" self.validator._validate_environment_exists() # Should only validate the environment mappings, not the basic values assert self.validator.errors == [] def test_environment_mapping_vs_basic_values_mismatch(self): """Test environment mismatch only in fields with environment mappings.""" self.validator.config = { "core": { "workspace_id": {"dev": "dev-id"}, # Environment mapping - missing 'prod' "repository_directory": "/single/path", # Basic value - should be ignored "item_types_in_scope": ["Notebook"], # Basic value - should be ignored }, "publish": { "exclude_regex": "^TEST.*", # Basic value - should be ignored "skip": {"dev": True}, # Environment mapping - missing 'prod' (optional, no error) }, } self.validator.environment = "prod" self.validator._validate_environment_exists() # Should get error only for required field with environment mapping # Optional fields (skip) only log debug, not errors assert len(self.validator.errors) == 1 error_text = " ".join(self.validator.errors) assert "workspace_id" in error_text assert "skip" not in error_text # Optional field should not cause error def test_environment_mismatch_optional_fields_log_only(self): """Test that optional fields only log warnings/debug when environment is missing.""" import logging self.validator.config = { "core": { "workspace_id": "simple-id", # Not a mapping, won't trigger error "repository_directory": "/path", "item_types_in_scope": {"dev": ["Notebook"]}, # Optional, warn_if_missing=True "parameter": {"dev": "param.yml"}, # Optional, warn_if_missing=True }, "publish": { "exclude_regex": {"dev": "^TEST.*"}, # Optional, warn_if_missing=False "skip": {"dev": True}, # Optional, warn_if_missing=False }, } self.validator.environment = "prod" with ( patch.object(logging.getLogger("fabric_cicd._common._config_validator"), "warning") as mock_warning, patch.object(logging.getLogger("fabric_cicd._common._config_validator"), "debug") as mock_debug, ): self.validator._validate_environment_exists() # No errors for optional fields assert self.validator.errors == [] # Warnings should be logged for item_types_in_scope and parameter assert mock_warning.call_count == 2 # Debug should be logged for exclude_regex and skip assert mock_debug.call_count == 2 def test_environment_mismatch_required_fields_error(self): """Test that required fields cause errors when environment is missing.""" self.validator.config = { "core": { "workspace_id": {"dev": "dev-id"}, # Required "repository_directory": {"dev": "/dev/path"}, # Required }, } self.validator.environment = "prod" self.validator._validate_environment_exists() # Should get errors for both required fields assert len(self.validator.errors) == 2 error_text = " ".join(self.validator.errors) assert "workspace_id" in error_text assert "repository_directory" in error_text def test_environment_exists_constants_per_key_env_missing(self): """Test _validate_environment_exists logs debug when env missing from per-key constants.""" import logging self.validator.config = { "core": { "workspace_id": "simple-id", "repository_directory": "/path", }, "constants": { "DEFAULT_API_ROOT_URL": { "dev": "https://api.fabric.microsoft.com", }, }, } self.validator.environment = "prod" with patch.object(logging.getLogger("fabric_cicd._common._config_validator"), "warning") as mock_warning: self.validator._validate_environment_exists() assert self.validator.errors == [] assert mock_warning.call_count == 1 assert "constants.DEFAULT_API_ROOT_URL" in mock_warning.call_args[0][0] def test_environment_exists_constants_per_key_env_present(self): """Test _validate_environment_exists passes when env exists in per-key constants.""" self.validator.config = { "core": { "workspace_id": {"dev": "dev-id"}, "repository_directory": "/path", }, "constants": { "DEFAULT_API_ROOT_URL": { "dev": "https://api.fabric.microsoft.com", }, }, } self.validator.environment = "dev" self.validator._validate_environment_exists() assert self.validator.errors == [] def test_environment_exists_constants_flat_value_no_env_check(self): """Test _validate_environment_exists skips flat constants (no env mapping to check).""" self.validator.config = { "core": { "workspace_id": "simple-id", "repository_directory": "/path", }, "constants": { "DEFAULT_API_ROOT_URL": "https://api.fabric.microsoft.com", }, } self.validator.environment = "prod" self.validator._validate_environment_exists() assert self.validator.errors == [] def test_environment_na_with_constants_env_mapping_no_error(self): """Test N/A environment does not flag constants per-key env mappings.""" self.validator.config = { "core": { "workspace_id": "simple-id", "repository_directory": "/path", }, "constants": { "DEFAULT_API_ROOT_URL": { "dev": "https://api.fabric.microsoft.com", }, }, } self.validator.environment = "N/A" self.validator._validate_environment_exists() # Constants per-key dicts are excluded from the N/A check assert self.validator.errors == [] ================================================ FILE: tests/test_deploy_with_config.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Tests for the config-based deployment functionality.""" from pathlib import Path from unittest.mock import MagicMock, patch import pytest import yaml from fabric_cicd import DeploymentResult, DeploymentStatus, constants, deploy_with_config from fabric_cicd._common._config_utils import ( config_overrides_scope, extract_publish_settings, extract_unpublish_settings, extract_workspace_settings, load_config_file, ) from fabric_cicd._common._config_validator import ConfigValidationError from fabric_cicd._common._exceptions import InputError, PublishError class TestConfigFileLoading: """Test config file loading and validation.""" def test_load_valid_config_file(self, tmp_path): """Test loading a valid YAML config file.""" # Create the actual directory structure that the config references test_repo_dir = tmp_path / "test" / "path" test_repo_dir.mkdir(parents=True) config_data = { "core": { "workspace_id": {"dev": "12345678-1234-1234-1234-123456789abc"}, "repository_directory": "test/path", } } config_file = tmp_path / "config.yml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) result = load_config_file(str(config_file), "dev") # Verify the structure is correct assert result["core"]["workspace_id"] == config_data["core"]["workspace_id"] # Verify path was resolved to absolute path and exists resolved_path = Path(result["core"]["repository_directory"]) assert resolved_path.is_absolute() assert resolved_path.exists() assert resolved_path.is_dir() def test_load_config_file_with_override(self, tmp_path): """Test loading a YAML config file with overrides.""" # Create the actual directory structure that the config references test_repo_dir = tmp_path / "test" / "path" test_repo_dir.mkdir(parents=True) config_data = { "core": { "workspace_id": {"dev": "12345678-1234-1234-1234-123456789abc"}, "repository_directory": "test/path", } } config_file = tmp_path / "config.yml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) # Define override values config_override = { "core": {"workspace_id": {"dev": "87654321-4321-4321-4321-123456789abc"}}, "publish": {"skip": False, "exclude_regex": "^TEST.*"}, } result = load_config_file(str(config_file), "dev", config_override) # Verify the overridden values assert result["core"]["workspace_id"]["dev"] == "87654321-4321-4321-4321-123456789abc" assert result["publish"]["skip"] == False assert result["publish"]["exclude_regex"] == "^TEST.*" # Verify path was still resolved to absolute path and exists resolved_path = Path(result["core"]["repository_directory"]) assert resolved_path.is_absolute() assert resolved_path.exists() assert resolved_path.is_dir() def test_load_nonexistent_config_file(self): """Test loading a non-existent config file raises ConfigValidationError.""" with pytest.raises(ConfigValidationError, match="Configuration file not found"): load_config_file("nonexistent.yml", "N/A") def test_load_invalid_yaml_syntax(self, tmp_path): """Test loading a file with invalid YAML syntax raises InputError.""" config_file = tmp_path / "invalid.yml" config_file.write_text("invalid: yaml: content: [") with pytest.raises(InputError, match="Invalid YAML syntax"): load_config_file(str(config_file), "N/A") def test_load_non_dict_yaml(self, tmp_path): """Test loading a YAML file that doesn't contain a dictionary.""" config_file = tmp_path / "list.yml" config_file.write_text("- item1\n- item2") with pytest.raises(ConfigValidationError, match="Configuration must be a dictionary"): load_config_file(str(config_file), "N/A") def test_load_config_missing_core_section(self, tmp_path): """Test loading a config file without required 'core' section.""" config_data = {"publish": {"skip": {"dev": True}}} config_file = tmp_path / "no_core.yml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) with pytest.raises(ConfigValidationError, match="must contain a 'core' section"): load_config_file(str(config_file), "N/A") class TestWorkspaceSettingsExtraction: """Test workspace settings extraction from config.""" def test_extract_workspace_id_by_environment(self): """Test extracting workspace ID based on environment.""" config = { "core": { "workspace_id": { "dev": "11111111-1111-1111-1111-111111111111", "prod": "22222222-2222-2222-2222-222222222222", }, "repository_directory": "test/path", } } settings = extract_workspace_settings(config, "dev") assert settings["workspace_id"] == "11111111-1111-1111-1111-111111111111" assert settings["repository_directory"] == "test/path" def test_extract_workspace_name_by_environment(self): """Test extracting workspace name based on environment.""" config = { "core": { "workspace": {"dev": "dev-workspace", "prod": "prod-workspace"}, "repository_directory": "test/path", } } settings = extract_workspace_settings(config, "dev") assert settings["workspace_name"] == "dev-workspace" assert settings["repository_directory"] == "test/path" def test_extract_single_workspace_id(self, tmp_path): """Test config with single workspace ID (non-environment-specific).""" # Create the actual directory structure that the config references test_repo_dir = tmp_path / "test" / "path" test_repo_dir.mkdir(parents=True) config_data = { "core": { "workspace_id": "33333333-3333-3333-3333-333333333333", "repository_directory": "test/path", } } config_file = tmp_path / "config.yml" config_file.write_text(yaml.dump(config_data)) # Single workspace IDs are supported config = load_config_file(str(config_file), "N/A") assert config["core"]["workspace_id"] == "33333333-3333-3333-3333-333333333333" def test_extract_missing_environment(self, tmp_path): """Test error when environment not found in workspace mappings during config loading.""" config_data = { "core": { "workspace_id": {"dev": "44444444-4444-4444-4444-444444444444"}, "repository_directory": "test/path", } } config_file = tmp_path / "config.yml" config_file.write_text(yaml.dump(config_data)) # Environment validation should happen during config loading, not extraction with pytest.raises( ConfigValidationError, match=r"Environment 'prod' not found in 'core.workspace_id' mappings" ): load_config_file(str(config_file), "prod") def test_extract_missing_workspace_config(self, tmp_path): """Test error when neither workspace_id nor workspace is provided.""" config_data = { "core": { "repository_directory": "test/path", } } config_file = tmp_path / "config.yml" config_file.write_text(yaml.dump(config_data)) with pytest.raises(ConfigValidationError, match="must specify either 'workspace_id' or 'workspace'"): load_config_file(str(config_file), "N/A") def test_extract_missing_repository_directory(self, tmp_path): """Test error when repository_directory is missing.""" config_data = { "core": { "workspace_id": {"dev": "55555555-5555-5555-5555-555555555555"}, } } config_file = tmp_path / "config.yml" config_file.write_text(yaml.dump(config_data)) with pytest.raises(ConfigValidationError, match="must specify 'repository_directory'"): load_config_file(str(config_file), "N/A") def test_extract_optional_item_types(self): """Test extracting optional item_types_in_scope.""" config = { "core": { "workspace_id": "66666666-6666-6666-6666-666666666666", "repository_directory": "test/path", "item_types_in_scope": ["Notebook", "DataPipeline"], } } settings = extract_workspace_settings(config, "dev") assert settings["item_types_in_scope"] == ["Notebook", "DataPipeline"] def test_extract_parameter_file_path_string(self): """Test extracting parameter file path as string.""" config = { "core": { "workspace_id": "12345678-1234-1234-1234-123456789abc", "repository_directory": "test/path", "parameter": "parameter.yml", } } settings = extract_workspace_settings(config, "dev") assert settings["parameter_file_path"] == "parameter.yml" def test_extract_parameter_file_path_environment_mapping(self): """Test extracting parameter file path from environment mapping.""" config = { "core": { "workspace_id": { "dev": "11111111-1111-1111-1111-111111111111", "prod": "22222222-2222-2222-2222-222222222222", }, "repository_directory": "test/path", "parameter": {"dev": "dev-parameter.yml", "prod": "prod-parameter.yml"}, } } settings = extract_workspace_settings(config, "dev") assert settings["parameter_file_path"] == "dev-parameter.yml" settings_prod = extract_workspace_settings(config, "prod") assert settings_prod["parameter_file_path"] == "prod-parameter.yml" def test_extract_parameter_file_path_missing(self): """Test extracting workspace settings when parameter field is missing.""" config = { "core": { "workspace_id": "33333333-3333-3333-3333-333333333333", "repository_directory": "test/path", } } settings = extract_workspace_settings(config, "dev") assert "parameter_file_path" not in settings class TestPublishSettingsExtraction: """Test publish settings extraction from config.""" def testextract_publish_settings_with_skip(self): """Test extracting publish settings with environment-specific skip.""" config = { "publish": { "exclude_regex": "^DONT_DEPLOY.*", "skip": {"dev": True, "prod": False}, } } settings = extract_publish_settings(config, "dev") assert settings["exclude_regex"] == "^DONT_DEPLOY.*" assert settings["skip"] is True settings = extract_publish_settings(config, "prod") assert settings["skip"] is False def testextract_publish_settings_with_items_to_include(self): """Test extracting publish settings with items_to_include.""" config = { "publish": { "items_to_include": ["item1.Notebook", "item2.DataPipeline"], } } settings = extract_publish_settings(config, "dev") assert settings["items_to_include"] == ["item1.Notebook", "item2.DataPipeline"] def testextract_publish_settings_no_config(self): """Test extracting publish settings when no publish config exists.""" config = {} settings = extract_publish_settings(config, "dev") assert settings == {} def testextract_publish_settings_single_skip_value(self): """Test extracting publish settings with single skip value (not environment-specific).""" config = { "publish": { "skip": True, } } settings = extract_publish_settings(config, "dev") assert settings["skip"] is True def test_extract_publish_settings_with_shortcut_exclude_regex(self): """Test extracting publish settings with shortcut_exclude_regex.""" config = { "publish": { "shortcut_exclude_regex": "^temp_.*", } } settings = extract_publish_settings(config, "dev") assert settings["shortcut_exclude_regex"] == "^temp_.*" def test_extract_publish_settings_with_environment_specific_shortcut_exclude_regex(self): """Test extracting publish settings with environment-specific shortcut_exclude_regex.""" config = { "publish": { "shortcut_exclude_regex": {"dev": "^dev_temp_.*", "prod": "^staging_.*"}, } } settings = extract_publish_settings(config, "dev") assert settings["shortcut_exclude_regex"] == "^dev_temp_.*" settings = extract_publish_settings(config, "prod") assert settings["shortcut_exclude_regex"] == "^staging_.*" class TestUnpublishSettingsExtraction: """Test unpublish settings extraction from config.""" def testextract_unpublish_settings_with_skip(self): """Test extracting unpublish settings with environment-specific skip.""" config = { "unpublish": { "exclude_regex": "^DEBUG.*", "skip": {"dev": True, "prod": False}, } } settings = extract_unpublish_settings(config, "dev") assert settings["exclude_regex"] == "^DEBUG.*" assert settings["skip"] is True settings = extract_unpublish_settings(config, "prod") assert settings["skip"] is False def testextract_unpublish_settings_no_config(self): """Test extracting unpublish settings when no unpublish config exists.""" config = {} settings = extract_unpublish_settings(config, "dev") assert settings == {} class TestConfigOverrides: """Test feature flags and constants overrides.""" def test_feature_flags_applied_within_scope(self): """Test feature flags are active inside config_overrides_scope.""" foo = "enable_foo_feature" bar = "enable_bar_feature" config = {"features": [foo, bar]} original_flags = constants.FEATURE_FLAG.copy() with config_overrides_scope(config, "N/A"): assert foo in constants.FEATURE_FLAG assert bar in constants.FEATURE_FLAG # Verify flags are restored after exiting scope assert original_flags == constants.FEATURE_FLAG def test_feature_flags_restored_after_scope(self): """Test feature flags set by config do not persist after scope exits.""" config = {"features": ["enable_test_feature"]} original_flags = constants.FEATURE_FLAG.copy() with config_overrides_scope(config, "N/A"): assert "enable_test_feature" in constants.FEATURE_FLAG assert "enable_test_feature" not in constants.FEATURE_FLAG assert original_flags == constants.FEATURE_FLAG def test_constants_overrides_applied_within_scope(self): """Test constants overrides are active inside config_overrides_scope.""" original_url = constants.DEFAULT_API_ROOT_URL config = {"constants": {"DEFAULT_API_ROOT_URL": "https://custom.api.com"}} with config_overrides_scope(config, "N/A"): assert constants.DEFAULT_API_ROOT_URL == "https://custom.api.com" # Verify constant is restored after exiting scope assert original_url == constants.DEFAULT_API_ROOT_URL def test_constants_overrides_restored_after_scope(self): """Test constants set by config do not persist after scope exits.""" original_url = constants.DEFAULT_API_ROOT_URL config = {"constants": {"DEFAULT_API_ROOT_URL": "https://temporary.api.com"}} with config_overrides_scope(config, "N/A"): pass assert original_url == constants.DEFAULT_API_ROOT_URL def test_user_constants_preserved_across_scope(self): """Test that constants set by user before scope are preserved after scope exits.""" original_url = constants.DEFAULT_API_ROOT_URL constants.DEFAULT_API_ROOT_URL = "https://user-set.api.com" config = {"constants": {"FABRIC_API_ROOT_URL": "https://config-set.fabric.com"}} original_fabric_url = constants.FABRIC_API_ROOT_URL with config_overrides_scope(config, "N/A"): assert constants.DEFAULT_API_ROOT_URL == "https://user-set.api.com" assert constants.FABRIC_API_ROOT_URL == "https://config-set.fabric.com" assert constants.DEFAULT_API_ROOT_URL == "https://user-set.api.com" assert original_fabric_url == constants.FABRIC_API_ROOT_URL # Clean up constants.DEFAULT_API_ROOT_URL = original_url def test_user_constant_same_key_restored_after_scope(self): """Test that when user sets a constant and config overrides the same key, user value is restored.""" original_url = constants.DEFAULT_API_ROOT_URL constants.DEFAULT_API_ROOT_URL = "https://user-set.api.com" config = {"constants": {"DEFAULT_API_ROOT_URL": "https://config-override.api.com"}} with config_overrides_scope(config, "N/A"): assert constants.DEFAULT_API_ROOT_URL == "https://config-override.api.com" assert constants.DEFAULT_API_ROOT_URL == "https://user-set.api.com" # Clean up constants.DEFAULT_API_ROOT_URL = original_url def test_no_overrides_does_not_error(self): """Test config_overrides_scope works with empty config.""" original_flags = constants.FEATURE_FLAG.copy() config = {} with config_overrides_scope(config, "N/A"): pass assert original_flags == constants.FEATURE_FLAG def test_overrides_restored_on_exception(self): """Test that overrides are restored even when an exception occurs inside the scope.""" original_url = constants.DEFAULT_API_ROOT_URL original_flags = constants.FEATURE_FLAG.copy() config = { "features": ["enable_test_feature"], "constants": {"DEFAULT_API_ROOT_URL": "https://will-fail.api.com"}, } msg = "deployment failed" with pytest.raises(RuntimeError, match=msg), config_overrides_scope(config, "N/A"): raise RuntimeError(msg) assert original_url == constants.DEFAULT_API_ROOT_URL assert original_flags == constants.FEATURE_FLAG def test_user_flags_preserved_across_scope(self): """Test that flags set by user before scope are preserved after scope exits.""" original_flags = constants.FEATURE_FLAG.copy() constants.FEATURE_FLAG.add("user_set_flag") config = {"features": ["config_set_flag"]} with config_overrides_scope(config, "N/A"): assert "user_set_flag" in constants.FEATURE_FLAG assert "config_set_flag" in constants.FEATURE_FLAG assert "user_set_flag" in constants.FEATURE_FLAG assert "config_set_flag" not in constants.FEATURE_FLAG # Clean up constants.FEATURE_FLAG.clear() constants.FEATURE_FLAG.update(original_flags) def test_environment_specific_feature_flags(self): """Test environment-specific feature flags are resolved correctly.""" original_flags = constants.FEATURE_FLAG.copy() config = { "features": { "dev": ["enable_dev_feature"], "prod": ["enable_prod_feature"], } } with config_overrides_scope(config, "dev"): assert "enable_dev_feature" in constants.FEATURE_FLAG assert "enable_prod_feature" not in constants.FEATURE_FLAG assert original_flags == constants.FEATURE_FLAG def test_environment_specific_constants(self): """Test environment-specific constants overrides are resolved correctly.""" original_url = constants.DEFAULT_API_ROOT_URL config = { "constants": { "DEFAULT_API_ROOT_URL": { "dev": "https://dev.api.com", "prod": "https://prod.api.com", } } } with config_overrides_scope(config, "dev"): assert constants.DEFAULT_API_ROOT_URL == "https://dev.api.com" assert original_url == constants.DEFAULT_API_ROOT_URL class TestConfigOverridesIntegration: """Integration tests for config_overrides_scope with deploy_with_config.""" @patch("fabric_cicd.publish.FabricWorkspace") @patch("fabric_cicd.publish.publish_all_items") @patch("fabric_cicd.publish.unpublish_all_orphan_items") def test_response_collection_via_config_features(self, mock_unpublish, mock_publish, mock_workspace, tmp_path): """Test that enable_response_collection set in config features enables response collection.""" _ = mock_unpublish _ = mock_publish test_repo_dir = tmp_path / "test" / "path" test_repo_dir.mkdir(parents=True) config_data = { "core": { "workspace_id": {"dev": "77777777-7777-7777-7777-777777777777"}, "repository_directory": "test/path", }, "features": ["enable_response_collection"], } config_file = tmp_path / "config.yml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) mock_workspace_instance = MagicMock() mock_workspace_instance.responses = {"Notebook": {"MyNotebook": {"body": {"id": "123"}}}} mock_workspace_instance.unpublish_responses = None mock_workspace.return_value = mock_workspace_instance result = deploy_with_config(config_file_path=str(config_file), token_credential=MagicMock(), environment="dev") assert isinstance(result, DeploymentResult) assert result.responses == {"publish": {"Notebook": {"MyNotebook": {"body": {"id": "123"}}}}} # Verify feature flag was restored after scope exit assert "enable_response_collection" not in constants.FEATURE_FLAG @patch("fabric_cicd.publish.FabricWorkspace") @patch("fabric_cicd.publish.publish_all_items") @patch("fabric_cicd.publish.unpublish_all_orphan_items") def test_failure_with_partial_responses_via_config_features( self, mock_unpublish, mock_publish, mock_workspace, tmp_path ): """Test that partial responses are attached to exceptions when enable_response_collection is set via config.""" _ = mock_publish test_repo_dir = tmp_path / "test" / "path" test_repo_dir.mkdir(parents=True) config_data = { "core": { "workspace_id": {"dev": "77777777-7777-7777-7777-777777777777"}, "repository_directory": "test/path", }, "features": ["enable_response_collection"], } config_file = tmp_path / "config.yml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) mock_workspace_instance = MagicMock() mock_workspace_instance.responses = {"Notebook": {"MyNotebook": {"body": {"id": "123"}}}} mock_workspace_instance.unpublish_responses = None mock_workspace.return_value = mock_workspace_instance mock_unpublish.side_effect = RuntimeError("Unpublish failed") with pytest.raises(RuntimeError) as exc_info: deploy_with_config(config_file_path=str(config_file), token_credential=MagicMock(), environment="dev") e = exc_info.value assert hasattr(e, "deployment_result") assert e.deployment_result.responses == {"publish": {"Notebook": {"MyNotebook": {"body": {"id": "123"}}}}} # Verify feature flag was restored after scope exit assert "enable_response_collection" not in constants.FEATURE_FLAG class TestDeployWithConfig: """Test the main deploy_with_config function.""" @patch("fabric_cicd.publish.FabricWorkspace") @patch("fabric_cicd.publish.publish_all_items") @patch("fabric_cicd.publish.unpublish_all_orphan_items") def test_deploy_with_config_full_deployment(self, mock_unpublish, mock_publish, mock_workspace, tmp_path): """Test full deployment with config file.""" # Create the actual directory structure that the config references test_repo_dir = tmp_path / "test" / "path" test_repo_dir.mkdir(parents=True) # Create test config file config_data = { "core": { "workspace_id": {"dev": "77777777-7777-7777-7777-777777777777"}, "repository_directory": "test/path", "item_types_in_scope": ["Notebook", "DataPipeline"], }, "publish": { "exclude_regex": "^DONT_DEPLOY.*", "skip": {"dev": False}, }, "unpublish": { "exclude_regex": "^DEBUG.*", "skip": {"dev": False}, }, } config_file = tmp_path / "config.yml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) # Mock workspace instance mock_workspace_instance = MagicMock() mock_workspace.return_value = mock_workspace_instance mock_credential = MagicMock() # Execute deployment deploy_with_config(config_file_path=str(config_file), token_credential=mock_credential, environment="dev") # Verify workspace creation # Note: repository_directory will be resolved to absolute path during validation call_args = mock_workspace.call_args[1] assert call_args["workspace_id"] == "77777777-7777-7777-7777-777777777777" assert call_args["workspace_name"] is None assert "test" in call_args["repository_directory"] # Path will be resolved to absolute assert "path" in call_args["repository_directory"] assert call_args["item_type_in_scope"] == ["Notebook", "DataPipeline"] assert call_args["environment"] == "dev" assert call_args["token_credential"] == mock_credential # Verify publish and unpublish calls mock_publish.assert_called_once_with( mock_workspace_instance, item_name_exclude_regex="^DONT_DEPLOY.*", folder_path_exclude_regex=None, folder_path_to_include=None, items_to_include=None, shortcut_exclude_regex=None, ) mock_unpublish.assert_called_once_with( mock_workspace_instance, item_name_exclude_regex="^DEBUG.*", items_to_include=None, ) @patch("fabric_cicd.publish.FabricWorkspace") @patch("fabric_cicd.publish.publish_all_items") @patch("fabric_cicd.publish.unpublish_all_orphan_items") def test_deploy_with_config_skip_operations(self, mock_unpublish, mock_publish, mock_workspace, tmp_path): """Test deployment with skip flags enabled.""" # Create the actual directory structure that the config references test_repo_dir = tmp_path / "test" / "path" test_repo_dir.mkdir(parents=True) # Create test config file with skip flags config_data = { "core": { "workspace_id": {"dev": "88888888-8888-8888-8888-888888888888"}, "repository_directory": "test/path", }, "publish": { "skip": {"dev": True}, }, "unpublish": { "skip": {"dev": True}, }, } config_file = tmp_path / "config.yml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) # Mock workspace instance mock_workspace_instance = MagicMock() mock_workspace.return_value = mock_workspace_instance # Execute deployment deploy_with_config(config_file_path=str(config_file), token_credential=MagicMock(), environment="dev") # Verify workspace creation mock_workspace.assert_called_once() # Verify that publish and unpublish are NOT called due to skip flags mock_publish.assert_not_called() mock_unpublish.assert_not_called() def test_deploy_with_config_missing_file(self): """Test deployment with missing config file.""" with pytest.raises(ConfigValidationError, match="Configuration file not found"): deploy_with_config(config_file_path="nonexistent.yml", token_credential=MagicMock(), environment="dev") @patch("fabric_cicd.publish.FabricWorkspace") @patch("fabric_cicd.publish.publish_all_items") @patch("fabric_cicd.publish.unpublish_all_orphan_items") def test_deploy_with_config_with_token_credential(self, mock_unpublish, mock_publish, mock_workspace, tmp_path): """Test deployment with custom token credential.""" # Mark unused mocks to avoid linting warnings _ = mock_unpublish _ = mock_publish # Create the actual directory structure that the config references test_repo_dir = tmp_path / "test" / "path" test_repo_dir.mkdir(parents=True) # Create test config file config_data = { "core": { "workspace_id": {"dev": "99999999-9999-9999-9999-999999999999"}, "repository_directory": "test/path", }, } config_file = tmp_path / "config.yml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) # Mock workspace instance and token credential mock_workspace_instance = MagicMock() mock_workspace.return_value = mock_workspace_instance mock_credential = MagicMock() # Execute deployment deploy_with_config(config_file_path=str(config_file), token_credential=mock_credential, environment="dev") # Verify workspace creation with token credential # Note: repository_directory will be resolved to absolute path during validation call_args = mock_workspace.call_args[1] assert call_args["workspace_id"] == "99999999-9999-9999-9999-999999999999" assert call_args["workspace_name"] is None assert "test" in call_args["repository_directory"] # Path will be resolved to absolute assert "path" in call_args["repository_directory"] assert call_args["item_type_in_scope"] is None assert call_args["environment"] == "dev" assert call_args["token_credential"] == mock_credential @patch("fabric_cicd.publish.FabricWorkspace") @patch("fabric_cicd.publish.publish_all_items") @patch("fabric_cicd.publish.unpublish_all_orphan_items") def test_deploy_with_config_with_config_override(self, mock_unpublish, mock_publish, mock_workspace, tmp_path): """Test deployment with config override.""" # Create the actual directory structure that the config references test_repo_dir = tmp_path / "test" / "path" test_repo_dir.mkdir(parents=True) # Create test config file with default publish.skip = True to skip publishing config_data = { "core": { "workspace_id": {"dev": "12345678-1234-1234-1234-123456789abc"}, "repository_directory": "test/path", }, "publish": { "skip": {"dev": True}, }, "unpublish": { "skip": {"dev": True}, }, } config_file = tmp_path / "config.yml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) # Define config override to override the skip flags config_override = { "publish": {"skip": {"dev": False}}, # Override to NOT skip publish "unpublish": {"skip": {"dev": False}}, # Override to NOT skip unpublish } # Mock workspace instance mock_workspace_instance = MagicMock() mock_workspace.return_value = mock_workspace_instance # Execute deployment with config override deploy_with_config( config_file_path=str(config_file), token_credential=MagicMock(), environment="dev", config_override=config_override, ) # Verify workspace creation mock_workspace.assert_called_once() # Verify that publish and unpublish ARE called because the override turns off the skip flags mock_publish.assert_called_once() mock_unpublish.assert_called_once() @patch("fabric_cicd.publish.FabricWorkspace") @patch("fabric_cicd.publish.publish_all_items") @patch("fabric_cicd.publish.unpublish_all_orphan_items") def test_deploy_with_config_shortcut_exclude_regex(self, mock_unpublish, mock_publish, mock_workspace, tmp_path): """Test deployment with shortcut_exclude_regex in config.""" # Create the actual directory structure that the config references test_repo_dir = tmp_path / "test" / "path" test_repo_dir.mkdir(parents=True) # Create test config file with shortcut_exclude_regex config_data = { "core": { "workspace_id": "12345678-1234-1234-1234-123456789abc", "repository_directory": "test/path", "item_types_in_scope": ["Lakehouse"], }, "publish": { "shortcut_exclude_regex": "^temp_.*", }, } config_file = tmp_path / "config.yml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) # Mock workspace instance mock_workspace_instance = MagicMock() mock_workspace.return_value = mock_workspace_instance # Execute deployment deploy_with_config(config_file_path=str(config_file), token_credential=MagicMock(), environment="dev") # Verify publish was called with shortcut_exclude_regex parameter mock_publish.assert_called_once_with( mock_workspace_instance, item_name_exclude_regex=None, folder_path_exclude_regex=None, folder_path_to_include=None, items_to_include=None, shortcut_exclude_regex="^temp_.*", ) # Verify unpublish was also called (but without shortcut_exclude_regex since it's publish-only) mock_unpublish.assert_called_once() @patch("fabric_cicd.publish.FabricWorkspace") @patch("fabric_cicd.publish.publish_all_items") @patch("fabric_cicd.publish.unpublish_all_orphan_items") def test_folder_path_to_include_passed_to_publish(self, _mock_unpublish, mock_publish, mock_workspace, tmp_path): # noqa: PT019 """Test that folder_path_to_include from config is passed to publish_all_items.""" test_repo_dir = tmp_path / "repo" test_repo_dir.mkdir(parents=True) config_data = { "core": { "workspace_id": "11111111-1111-1111-1111-111111111111", "repository_directory": str(test_repo_dir), }, "publish": { "folder_path_to_include": ["/my/folder/path"], }, } config_file = tmp_path / "config.yml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) mock_workspace.return_value = MagicMock() deploy_with_config(config_file_path=str(config_file), token_credential=MagicMock(), environment="dev") call_args = mock_publish.call_args[1] assert call_args["folder_path_to_include"] == ["/my/folder/path"] @patch("fabric_cicd.publish.FabricWorkspace") @patch("fabric_cicd.publish.publish_all_items") @patch("fabric_cicd.publish.unpublish_all_orphan_items") def test_folder_path_to_include_defaults_to_none(self, _mock_unpublish, mock_publish, mock_workspace, tmp_path): # noqa: PT019 """Test that folder_path_to_include defaults to None when not specified.""" test_repo_dir = tmp_path / "repo" test_repo_dir.mkdir(parents=True) config_data = { "core": { "workspace_id": "11111111-1111-1111-1111-111111111111", "repository_directory": str(test_repo_dir), }, } config_file = tmp_path / "config.yml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) mock_workspace.return_value = MagicMock() deploy_with_config(config_file_path=str(config_file), token_credential=MagicMock(), environment="dev") call_args = mock_publish.call_args[1] assert call_args["folder_path_to_include"] is None @patch("fabric_cicd.publish.FabricWorkspace") @patch("fabric_cicd.publish.publish_all_items") @patch("fabric_cicd.publish.unpublish_all_orphan_items") def test_folder_path_to_include_environment_specific(self, _mock_unpublish, mock_publish, mock_workspace, tmp_path): # noqa: PT019 """Test that folder_path_to_include resolves environment-specific values.""" test_repo_dir = tmp_path / "repo" test_repo_dir.mkdir(parents=True) config_data = { "core": { "workspace_id": { "dev": "11111111-1111-1111-1111-111111111111", "prod": "22222222-2222-2222-2222-222222222222", }, "repository_directory": str(test_repo_dir), }, "publish": { "folder_path_to_include": {"dev": ["/dev/folder"], "prod": ["/prod/folder"]}, }, } config_file = tmp_path / "config.yml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) mock_workspace.return_value = MagicMock() deploy_with_config(config_file_path=str(config_file), token_credential=MagicMock(), environment="dev") call_args = mock_publish.call_args[1] assert call_args["folder_path_to_include"] == ["/dev/folder"] def test_deploy_with_config_skips_parameterization_when_parameter_absent(self, tmp_path): """Integration: deploy_with_config must not auto-discover parameter.yml when the 'parameter' field is absent from the config file.""" # Set up repo directory WITH a parameter.yml that would be auto-discovered repo_dir = tmp_path / "workspace" repo_dir.mkdir() (repo_dir / "parameter.yml").write_text( "find_replace:\n - find_value: 'old'\n replace_value:\n dev: 'new'\n" ) config_data = { "core": { "workspace_id": {"dev": "11111111-1111-1111-1111-111111111111"}, "repository_directory": str(repo_dir), # 'parameter' intentionally omitted } } config_file = tmp_path / "config.yml" config_file.write_text(yaml.dump(config_data)) with patch("fabric_cicd.publish.FabricWorkspace") as mock_fabric_ws: mock_ws = MagicMock() mock_ws.environment_parameter = {} mock_fabric_ws.return_value = mock_ws # Call deploy_with_config and verify skip_parameterization=True was passed # (mock publish/unpublish to avoid real API calls) with ( patch("fabric_cicd.publish.publish_all_items"), patch("fabric_cicd.publish.unpublish_all_orphan_items"), ): deploy_with_config(config_file_path=str(config_file), token_credential=MagicMock(), environment="dev") # Assert FabricWorkspace was constructed with skip_parameterization=True call_kwargs = mock_fabric_ws.call_args[1] assert call_kwargs.get("skip_parameterization") is True def test_deploy_with_config_loads_parameter_when_field_present(self, tmp_path): """Integration: deploy_with_config must pass skip_parameterization=False when the 'parameter' field IS present in config.""" repo_dir = tmp_path / "workspace" repo_dir.mkdir() config_data = { "core": { "workspace_id": {"dev": "11111111-1111-1111-1111-111111111111"}, "repository_directory": str(repo_dir), "parameter": "my-params.yml", } } config_file = tmp_path / "config.yml" config_file.write_text(yaml.dump(config_data)) parameter_file = tmp_path / "my-params.yml" parameter_file.write_text("find_replace:\n - find_value: 'old'\n replace_value:\n dev: 'new'\n") with patch("fabric_cicd.publish.FabricWorkspace") as mock_fabric_ws: mock_ws = MagicMock() mock_fabric_ws.return_value = mock_ws with ( patch("fabric_cicd.publish.publish_all_items"), patch("fabric_cicd.publish.unpublish_all_orphan_items"), ): deploy_with_config(config_file_path=str(config_file), token_credential=MagicMock(), environment="dev") call_kwargs = mock_fabric_ws.call_args[1] assert call_kwargs.get("skip_parameterization") is False class TestConfigIntegration: """Integration tests for config functionality.""" def test_sample_config_file_structure(self): """Test that the sample config file can be loaded and parsed correctly.""" # Test with the actual sample config file sample_config_path = Path(__file__).parent.parent / "sample" / "workspace" / "config.yml" if sample_config_path.exists(): # The sample config file might have directory references that don't exist in the test environment # So we just verify it can be parsed as valid YAML import yaml with sample_config_path.open(encoding="utf-8") as f: config = yaml.safe_load(f) # Verify basic structure assert "core" in config # If we have a valid environment, test basic functionality if ( "core" in config and "workspace_id" in config["core"] and isinstance(config["core"]["workspace_id"], dict) ): test_env = next(iter(config["core"]["workspace_id"].keys())) # Only test the config extraction without path validation workspace_settings = extract_workspace_settings(config, test_env) assert "repository_directory" in workspace_settings extract_publish_settings(config, test_env) extract_unpublish_settings(config, test_env) with config_overrides_scope(config, test_env): pass def test_config_validation_comprehensive(self, tmp_path): """Test comprehensive config validation with all sections.""" # Create the actual directory structure that the config references sample_workspace_dir = tmp_path / "sample" / "workspace" sample_workspace_dir.mkdir(parents=True) config_data = { "core": { "workspace_id": { "dev": "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", "test": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb", "prod": "cccccccc-cccc-cccc-cccc-cccccccccccc", }, "repository_directory": "sample/workspace", "item_types_in_scope": ["Environment", "Notebook", "DataPipeline"], }, "publish": { "exclude_regex": "^DONT_DEPLOY.*", "items_to_include": ["item1.Notebook"], "skip": {"dev": True, "test": False, "prod": False}, }, "unpublish": {"exclude_regex": "^DEBUG.*", "skip": {"dev": True, "test": False, "prod": False}}, "features": ["enable_shortcut_publish"], "constants": {"DEFAULT_API_ROOT_URL": "https://msitapi.fabric.microsoft.com"}, } config_file = tmp_path / "comprehensive_config.yml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) # Test loading and parsing config = load_config_file(str(config_file), "dev") # Config validation may modify the config (e.g., resolve paths) # So we test the important parts separately assert "core" in config assert config["core"]["workspace_id"] == config_data["core"]["workspace_id"] assert "Notebook" in config["core"]["item_types_in_scope"] assert "publish" in config assert config["publish"]["exclude_regex"] == config_data["publish"]["exclude_regex"] # Test all environment extractions for env in ["dev", "test", "prod"]: workspace_settings = extract_workspace_settings(config, env) assert workspace_settings["workspace_id"] == config_data["core"]["workspace_id"][env] publish_settings = extract_publish_settings(config, env) assert publish_settings["skip"] == config_data["publish"]["skip"][env] unpublish_settings = extract_unpublish_settings(config, env) assert unpublish_settings["skip"] == config_data["unpublish"]["skip"][env] class TestConfigUtilsExtractSettings: """Test config utility functions for extracting settings.""" def test_extract_publish_settings_with_folder_exclude_regex(self): """Test extracting publish settings with folder_exclude_regex.""" config = { "publish": { "folder_exclude_regex": "^/DONT_DEPLOY_FOLDER", } } settings = extract_publish_settings(config, "dev") assert settings["folder_exclude_regex"] == "^/DONT_DEPLOY_FOLDER" def test_extract_publish_settings_with_environment_specific_folder_exclude_regex(self): """Test extracting publish settings with environment-specific folder_exclude_regex.""" config = { "publish": { "folder_exclude_regex": {"dev": "^/DEV_FOLDER", "prod": "^/PROD_FOLDER"}, } } settings = extract_publish_settings(config, "dev") assert settings["folder_exclude_regex"] == "^/DEV_FOLDER" settings = extract_publish_settings(config, "prod") assert settings["folder_exclude_regex"] == "^/PROD_FOLDER" def test_extract_publish_settings_missing_environment_skips_setting(self): """Test that missing environment in optional publish settings skips the setting.""" config = { "publish": { "exclude_regex": {"dev": "^DEV.*"}, # Only dev defined "folder_exclude_regex": {"dev": "^/DEV_FOLDER"}, # Only dev defined } } # prod environment not defined - settings should be skipped settings = extract_publish_settings(config, "prod") assert "exclude_regex" not in settings assert "folder_exclude_regex" not in settings def test_extract_unpublish_settings_missing_environment_skips_setting(self): """Test that missing environment in optional unpublish settings skips the setting.""" config = { "unpublish": { "exclude_regex": {"dev": "^DEV.*"}, # Only dev defined "items_to_include": {"dev": ["item1"]}, # Only dev defined } } # prod environment not defined - settings should be skipped settings = extract_unpublish_settings(config, "prod") assert "exclude_regex" not in settings assert "items_to_include" not in settings def test_extract_publish_settings_skip_defaults_false_when_env_missing(self): """Test that skip defaults to False when environment is not in skip mapping.""" config = { "publish": { "skip": {"dev": True}, # Only dev defined } } # prod environment not defined - skip should default to False settings = extract_publish_settings(config, "prod") assert settings["skip"] is False def test_extract_unpublish_settings_skip_defaults_false_when_env_missing(self): """Test that skip defaults to False when environment is not in skip mapping.""" config = { "unpublish": { "skip": {"dev": True}, # Only dev defined } } # prod environment not defined - skip should default to False settings = extract_unpublish_settings(config, "prod") assert settings["skip"] is False def test_extract_workspace_settings_optional_fields_missing_environment(self): """Test that optional workspace fields are skipped when environment is missing.""" config = { "core": { "workspace_id": "12345678-1234-1234-1234-123456789abc", # Simple value "repository_directory": "/path/to/repo", "item_types_in_scope": {"dev": ["Notebook"]}, # Only dev defined "parameter": {"dev": "dev-param.yml"}, # Only dev defined } } # prod environment not defined for optional fields - they should be skipped settings = extract_workspace_settings(config, "prod") assert "item_types_in_scope" not in settings assert "parameter_file_path" not in settings # Required fields should still be present assert settings["workspace_id"] == "12345678-1234-1234-1234-123456789abc" assert settings["repository_directory"] == "/path/to/repo" def test_extract_publish_settings_shortcut_exclude_regex_missing_environment(self): """Test that shortcut_exclude_regex is skipped when environment is missing.""" config = { "publish": { "shortcut_exclude_regex": {"dev": "^dev_temp_.*"}, # Only dev defined } } # prod environment not defined - setting should be skipped settings = extract_publish_settings(config, "prod") assert "shortcut_exclude_regex" not in settings def test_extract_publish_settings_items_to_include_missing_environment(self): """Test that items_to_include is skipped when environment is missing.""" config = { "publish": { "items_to_include": {"dev": ["item1.Notebook", "item2.DataPipeline"]}, # Only dev defined } } # prod environment not defined - setting should be skipped settings = extract_publish_settings(config, "prod") assert "items_to_include" not in settings def test_extract_publish_settings_folder_path_to_include_list(self): """Test extract_publish_settings returns folder_path_to_include as a list.""" config = { "publish": { "folder_path_to_include": ["/my/folder/path"], }, } result = extract_publish_settings(config, "dev") assert result["folder_path_to_include"] == ["/my/folder/path"] def test_extract_publish_settings_folder_path_to_include_env_specific(self): """Test extract_publish_settings resolves folder_path_to_include per environment.""" config = { "publish": { "folder_path_to_include": {"dev": ["/dev/folder"], "prod": ["/prod/folder"]}, }, } result = extract_publish_settings(config, "dev") assert result["folder_path_to_include"] == ["/dev/folder"] def test_extract_publish_settings_folder_path_to_include_missing(self): """Test extract_publish_settings defaults folder_path_to_include to None.""" config = { "publish": { "exclude_regex": "^SKIP.*", }, } settings = extract_publish_settings(config, "dev") assert "folder_path_to_include" not in settings def test_extract_publish_settings_no_publish_section_folder_path_to_include(self): """Test extract_publish_settings defaults folder_path_to_include to None when no publish section.""" config = {} settings = extract_publish_settings(config, "dev") assert "folder_path_to_include" not in settings class TestGetConfigValue: """Test the get_config_value utility function.""" def test_get_config_value_key_not_present(self): """Test get_config_value when key doesn't exist.""" from fabric_cicd._common._config_utils import get_config_value config = {"other_key": "value"} result = get_config_value(config, "missing_key", "dev") assert result is None def test_get_config_value_simple_value(self): """Test get_config_value with simple (non-dict) value.""" from fabric_cicd._common._config_utils import get_config_value config = {"key": "simple_value"} result = get_config_value(config, "key", "dev") assert result == "simple_value" def test_get_config_value_dict_with_environment(self): """Test get_config_value with dict containing target environment.""" from fabric_cicd._common._config_utils import get_config_value config = {"key": {"dev": "dev_value", "prod": "prod_value"}} result = get_config_value(config, "key", "dev") assert result == "dev_value" def test_get_config_value_dict_missing_environment(self): """Test get_config_value with dict missing target environment.""" from fabric_cicd._common._config_utils import get_config_value config = {"key": {"dev": "dev_value"}} result = get_config_value(config, "key", "prod") assert result is None def test_get_config_value_list_value(self): """Test get_config_value with list value.""" from fabric_cicd._common._config_utils import get_config_value config = {"key": ["item1", "item2"]} result = get_config_value(config, "key", "dev") assert result == ["item1", "item2"] def test_get_config_value_bool_value(self): """Test get_config_value with boolean value.""" from fabric_cicd._common._config_utils import get_config_value config = {"key": True} result = get_config_value(config, "key", "dev") assert result is True class TestDeploymentResult: """Test DeploymentResult and DeploymentStatus types.""" def test_deployment_status_completed_value(self): """Test DeploymentStatus.COMPLETED has expected value.""" assert DeploymentStatus.COMPLETED.value == "completed" def test_deployment_status_failed_value(self): """Test DeploymentStatus.FAILED has expected value.""" assert DeploymentStatus.FAILED.value == "failed" def test_deployment_status_string_comparison(self): """Test DeploymentStatus supports string comparison via str inheritance.""" assert DeploymentStatus.COMPLETED == "completed" assert DeploymentStatus.FAILED == "failed" def test_deployment_result_structure(self): """Test DeploymentResult fields are set correctly.""" result = DeploymentResult( status=DeploymentStatus.COMPLETED, message="Test message", ) assert result.status == DeploymentStatus.COMPLETED assert result.message == "Test message" def test_deployment_result_responses_defaults_to_none(self): """Test DeploymentResult.responses defaults to None when not provided.""" result = DeploymentResult( status=DeploymentStatus.COMPLETED, message="Test message", ) assert result.responses is None def test_deployment_result_with_responses(self): """Test DeploymentResult stores responses when provided.""" responses = {"Notebook": {"MyNotebook": {"body": {"id": "123"}}}} result = DeploymentResult( status=DeploymentStatus.FAILED, message="Deployment failed", responses=responses, ) assert result.status == DeploymentStatus.FAILED assert result.message == "Deployment failed" assert result.responses == responses class TestDeployWithConfigReturnValue: """Test deploy_with_config returns DeploymentResult.""" @patch("fabric_cicd.publish.FabricWorkspace") @patch("fabric_cicd.publish.publish_all_items") @patch("fabric_cicd.publish.unpublish_all_orphan_items") @patch("fabric_cicd.constants.FEATURE_FLAG", set(["enable_experimental_features", "enable_config_deploy"])) def test_deploy_with_config_returns_deployment_result(self, mock_unpublish, mock_publish, mock_workspace, tmp_path): """Test that deploy_with_config returns a DeploymentResult on success.""" # Mark unused mocks to avoid linting warnings _ = mock_unpublish _ = mock_publish # Create the actual directory structure that the config references test_repo_dir = tmp_path / "test" / "path" test_repo_dir.mkdir(parents=True) # Create test config file config_data = { "core": { "workspace_id": {"dev": "77777777-7777-7777-7777-777777777777"}, "repository_directory": "test/path", }, } config_file = tmp_path / "config.yml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) # Mock workspace instance mock_workspace_instance = MagicMock() mock_workspace.return_value = mock_workspace_instance # Execute deployment result = deploy_with_config(config_file_path=str(config_file), token_credential=MagicMock(), environment="dev") # Verify result is a DeploymentResult assert isinstance(result, DeploymentResult) assert result.status == DeploymentStatus.COMPLETED assert "completed successfully" in result.message @patch("fabric_cicd.publish.FabricWorkspace") @patch("fabric_cicd.publish.publish_all_items") @patch("fabric_cicd.publish.unpublish_all_orphan_items") @patch("fabric_cicd.constants.FEATURE_FLAG", set(["enable_experimental_features", "enable_config_deploy"])) def test_deploy_with_config_returns_completed_when_skipping_operations( self, mock_unpublish, mock_publish, mock_workspace, tmp_path ): """Test that deploy_with_config returns COMPLETED status even when skipping operations.""" # Create the actual directory structure that the config references test_repo_dir = tmp_path / "test" / "path" test_repo_dir.mkdir(parents=True) # Create test config file with skip flags config_data = { "core": { "workspace_id": {"dev": "88888888-8888-8888-8888-888888888888"}, "repository_directory": "test/path", }, "publish": { "skip": {"dev": True}, }, "unpublish": { "skip": {"dev": True}, }, } config_file = tmp_path / "config.yml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) # Mock workspace instance mock_workspace_instance = MagicMock() mock_workspace.return_value = mock_workspace_instance # Execute deployment result = deploy_with_config(config_file_path=str(config_file), token_credential=MagicMock(), environment="dev") # Verify result is a DeploymentResult with COMPLETED status assert isinstance(result, DeploymentResult) assert result.status == DeploymentStatus.COMPLETED # Verify that publish and unpublish are NOT called due to skip flags mock_publish.assert_not_called() mock_unpublish.assert_not_called() class TestDeployWithConfigFailures: """Test deploy_with_config raises exceptions properly on failure.""" def test_deploy_with_config_invalid_yaml_raises_input_error(self, tmp_path): """Test that deploy_with_config raises InputError for invalid YAML syntax.""" config_file = tmp_path / "invalid.yml" config_file.write_text("invalid: yaml: content: [") with pytest.raises(InputError, match="Invalid YAML syntax"): deploy_with_config(config_file_path=str(config_file), token_credential=MagicMock(), environment="dev") def test_deploy_with_config_missing_core_raises_config_validation_error(self, tmp_path): """Test that deploy_with_config raises ConfigValidationError when core section is missing.""" config_data = {"publish": {"skip": True}} config_file = tmp_path / "no_core.yml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) with pytest.raises(ConfigValidationError, match="must contain a 'core' section"): deploy_with_config(config_file_path=str(config_file), token_credential=MagicMock(), environment="dev") def test_deploy_with_config_missing_environment_raises_config_validation_error(self, tmp_path): """Test that deploy_with_config raises ConfigValidationError when environment is not in workspace mappings.""" test_repo_dir = tmp_path / "test" / "path" test_repo_dir.mkdir(parents=True) config_data = { "core": { "workspace_id": {"dev": "12345678-1234-1234-1234-123456789abc"}, "repository_directory": "test/path", } } config_file = tmp_path / "config.yml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) with pytest.raises(ConfigValidationError, match="Environment 'prod' not found"): deploy_with_config(config_file_path=str(config_file), token_credential=MagicMock(), environment="prod") def test_deploy_with_config_missing_workspace_id_raises_config_validation_error(self, tmp_path): """Test that deploy_with_config raises ConfigValidationError when workspace_id is missing.""" config_data = { "core": { "repository_directory": "test/path", } } config_file = tmp_path / "config.yml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) with pytest.raises(ConfigValidationError, match="must specify either 'workspace_id' or 'workspace'"): deploy_with_config(config_file_path=str(config_file), token_credential=MagicMock(), environment="dev") @patch("fabric_cicd.publish.FabricWorkspace") @patch("fabric_cicd.publish.publish_all_items") @patch("fabric_cicd.publish.unpublish_all_orphan_items") def test_deploy_with_config_publish_error_propagates(self, mock_unpublish, mock_publish, mock_workspace, tmp_path): """Test that PublishError from publish_all_items propagates through deploy_with_config.""" _ = mock_unpublish test_repo_dir = tmp_path / "test" / "path" test_repo_dir.mkdir(parents=True) config_data = { "core": { "workspace_id": {"dev": "77777777-7777-7777-7777-777777777777"}, "repository_directory": "test/path", }, } config_file = tmp_path / "config.yml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) mock_workspace.return_value = MagicMock() mock_publish.side_effect = PublishError( errors=[("FailedNotebook", RuntimeError("API call failed"))], logger=MagicMock(), ) with pytest.raises(PublishError, match="Failed to publish 1 item"): deploy_with_config(config_file_path=str(config_file), token_credential=MagicMock(), environment="dev") @patch("fabric_cicd.publish.FabricWorkspace") @patch("fabric_cicd.publish.publish_all_items") @patch("fabric_cicd.publish.unpublish_all_orphan_items") def test_deploy_with_config_workspace_creation_error_propagates( self, mock_unpublish, mock_publish, mock_workspace, tmp_path ): """Test that exceptions from FabricWorkspace constructor propagate through deploy_with_config.""" _ = mock_unpublish _ = mock_publish test_repo_dir = tmp_path / "test" / "path" test_repo_dir.mkdir(parents=True) config_data = { "core": { "workspace_id": {"dev": "77777777-7777-7777-7777-777777777777"}, "repository_directory": "test/path", }, } config_file = tmp_path / "config.yml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) mock_workspace.side_effect = Exception("Workspace initialization failed") with pytest.raises(Exception, match="Workspace initialization failed"): deploy_with_config(config_file_path=str(config_file), token_credential=MagicMock(), environment="dev") mock_publish.assert_not_called() mock_unpublish.assert_not_called() @patch("fabric_cicd.publish.FabricWorkspace") @patch("fabric_cicd.publish.publish_all_items") @patch("fabric_cicd.publish.unpublish_all_orphan_items") def test_deploy_with_config_unpublish_error_propagates( self, mock_unpublish, mock_publish, mock_workspace, tmp_path ): """Test that exceptions from unpublish_all_orphan_items propagate and publish was already called.""" test_repo_dir = tmp_path / "test" / "path" test_repo_dir.mkdir(parents=True) config_data = { "core": { "workspace_id": {"dev": "77777777-7777-7777-7777-777777777777"}, "repository_directory": "test/path", }, } config_file = tmp_path / "config.yml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) mock_workspace.return_value = MagicMock() mock_unpublish.side_effect = RuntimeError("Unpublish operation failed") with pytest.raises(RuntimeError, match="Unpublish operation failed"): deploy_with_config(config_file_path=str(config_file), token_credential=MagicMock(), environment="dev") # Verify publish was called successfully before unpublish failed mock_publish.assert_called_once() class TestDeployWithConfigExceptionAttributes: """Test that deploy_with_config attaches deployment attributes to exceptions.""" @patch("fabric_cicd.publish.FabricWorkspace") @patch("fabric_cicd.publish.publish_all_items") @patch("fabric_cicd.publish.unpublish_all_orphan_items") def test_exception_has_deployment_status_and_message(self, mock_unpublish, mock_publish, mock_workspace, tmp_path): """Test that raised exceptions have deployment_status and deployment_message attributes.""" _ = mock_unpublish test_repo_dir = tmp_path / "test" / "path" test_repo_dir.mkdir(parents=True) config_data = { "core": { "workspace_id": {"dev": "77777777-7777-7777-7777-777777777777"}, "repository_directory": "test/path", }, } config_file = tmp_path / "config.yml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) mock_workspace.return_value = MagicMock() mock_publish.side_effect = RuntimeError("Something broke") with pytest.raises(RuntimeError) as exc_info: deploy_with_config(config_file_path=str(config_file), token_credential=MagicMock(), environment="dev") e = exc_info.value assert hasattr(e, "deployment_result") assert e.deployment_result.status == DeploymentStatus.FAILED assert "Something broke" in e.deployment_result.message @patch("fabric_cicd.publish.FabricWorkspace") @patch("fabric_cicd.publish.publish_all_items") @patch("fabric_cicd.publish.unpublish_all_orphan_items") @patch( "fabric_cicd.constants.FEATURE_FLAG", set(["enable_experimental_features", "enable_config_deploy", "enable_response_collection"]), ) def test_exception_has_partial_responses_when_enabled(self, mock_unpublish, mock_publish, mock_workspace, tmp_path): """Test that partial responses are attached to exceptions when response collection is enabled.""" _ = mock_publish test_repo_dir = tmp_path / "test" / "path" test_repo_dir.mkdir(parents=True) config_data = { "core": { "workspace_id": {"dev": "77777777-7777-7777-7777-777777777777"}, "repository_directory": "test/path", }, } config_file = tmp_path / "config.yml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) mock_workspace_instance = MagicMock() mock_workspace_instance.responses = {"Notebook": {"MyNotebook": {"body": {"id": "123"}}}} mock_workspace_instance.unpublish_responses = None mock_workspace.return_value = mock_workspace_instance mock_unpublish.side_effect = RuntimeError("Unpublish failed") with pytest.raises(RuntimeError) as exc_info: deploy_with_config(config_file_path=str(config_file), token_credential=MagicMock(), environment="dev") e = exc_info.value assert hasattr(e, "deployment_result") assert e.deployment_result.responses == {"publish": {"Notebook": {"MyNotebook": {"body": {"id": "123"}}}}} @patch("fabric_cicd.publish.FabricWorkspace") @patch("fabric_cicd.publish.publish_all_items") @patch("fabric_cicd.publish.unpublish_all_orphan_items") def test_exception_no_responses_when_flag_disabled(self, mock_unpublish, mock_publish, mock_workspace, tmp_path): """Test that responses are not attached to exceptions when response collection is disabled.""" _ = mock_unpublish test_repo_dir = tmp_path / "test" / "path" test_repo_dir.mkdir(parents=True) config_data = { "core": { "workspace_id": {"dev": "77777777-7777-7777-7777-777777777777"}, "repository_directory": "test/path", }, } config_file = tmp_path / "config.yml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) mock_workspace.return_value = MagicMock() mock_publish.side_effect = RuntimeError("Publish failed") with pytest.raises(RuntimeError) as exc_info: deploy_with_config(config_file_path=str(config_file), token_credential=MagicMock(), environment="dev") e = exc_info.value assert hasattr(e, "deployment_result") assert e.deployment_result.responses is None def test_pre_workspace_failure_has_deployment_attributes(self, tmp_path): """Test that failures before workspace creation still have deployment attributes.""" config_file = tmp_path / "invalid.yml" config_file.write_text("invalid: yaml: content: [") with pytest.raises(InputError) as exc_info: deploy_with_config(config_file_path=str(config_file), token_credential=MagicMock(), environment="dev") e = exc_info.value assert hasattr(e, "deployment_result") assert e.deployment_result.status == DeploymentStatus.FAILED assert e.deployment_result.message is not None assert e.deployment_result.responses is None class TestDeployWithConfigResponseCollection: """Test deploy_with_config response collection integration.""" @patch("fabric_cicd.publish.FabricWorkspace") @patch("fabric_cicd.publish.publish_all_items") @patch("fabric_cicd.publish.unpublish_all_orphan_items") @patch( "fabric_cicd.constants.FEATURE_FLAG", set(["enable_experimental_features", "enable_config_deploy", "enable_response_collection"]), ) def test_result_responses_is_dict_when_enabled(self, mock_unpublish, mock_publish, mock_workspace, tmp_path): """Test that result.responses is a dict (not string) when response collection is enabled.""" _ = mock_unpublish _ = mock_publish test_repo_dir = tmp_path / "test" / "path" test_repo_dir.mkdir(parents=True) config_data = { "core": { "workspace_id": {"dev": "77777777-7777-7777-7777-777777777777"}, "repository_directory": "test/path", }, } config_file = tmp_path / "config.yml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) mock_workspace_instance = MagicMock() mock_workspace_instance.responses = {"Notebook": {"MyNotebook": {"body": {"id": "123"}}}} mock_workspace_instance.unpublish_responses = None mock_workspace.return_value = mock_workspace_instance result = deploy_with_config(config_file_path=str(config_file), token_credential=MagicMock(), environment="dev") assert isinstance(result, DeploymentResult) assert isinstance(result.responses, dict) assert result.responses == {"publish": {"Notebook": {"MyNotebook": {"body": {"id": "123"}}}}} @patch("fabric_cicd.publish.FabricWorkspace") @patch("fabric_cicd.publish.publish_all_items") @patch("fabric_cicd.publish.unpublish_all_orphan_items") @patch( "fabric_cicd.constants.FEATURE_FLAG", set(["enable_experimental_features", "enable_config_deploy", "enable_response_collection"]), ) def test_result_responses_contains_both_publish_and_unpublish( self, mock_unpublish, mock_publish, mock_workspace, tmp_path ): """Test that result.responses contains both publish and unpublish keys when both are collected.""" _ = mock_unpublish _ = mock_publish test_repo_dir = tmp_path / "test" / "path" test_repo_dir.mkdir(parents=True) config_data = { "core": { "workspace_id": {"dev": "77777777-7777-7777-7777-777777777777"}, "repository_directory": "test/path", }, } config_file = tmp_path / "config.yml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) mock_workspace_instance = MagicMock() mock_workspace_instance.responses = {"Notebook": {"nb1": {"body": {"id": "123"}}}} mock_workspace_instance.unpublish_responses = {"Notebook": {"nb2": {"body": {"id": "456"}}}} mock_workspace.return_value = mock_workspace_instance result = deploy_with_config(config_file_path=str(config_file), token_credential=MagicMock(), environment="dev") assert isinstance(result, DeploymentResult) assert result.responses == { "publish": {"Notebook": {"nb1": {"body": {"id": "123"}}}}, "unpublish": {"Notebook": {"nb2": {"body": {"id": "456"}}}}, } @patch("fabric_cicd.publish.FabricWorkspace") @patch("fabric_cicd.publish.publish_all_items") @patch("fabric_cicd.publish.unpublish_all_orphan_items") @patch( "fabric_cicd.constants.FEATURE_FLAG", set(["enable_experimental_features", "enable_config_deploy"]), ) def test_result_responses_is_none_when_disabled(self, mock_unpublish, mock_publish, mock_workspace, tmp_path): """Test that result.responses is None when response collection is not enabled.""" _ = mock_unpublish _ = mock_publish test_repo_dir = tmp_path / "test" / "path" test_repo_dir.mkdir(parents=True) config_data = { "core": { "workspace_id": {"dev": "77777777-7777-7777-7777-777777777777"}, "repository_directory": "test/path", }, } config_file = tmp_path / "config.yml" with Path.open(config_file, "w") as f: yaml.dump(config_data, f) mock_workspace_instance = MagicMock() mock_workspace.return_value = mock_workspace_instance result = deploy_with_config(config_file_path=str(config_file), token_credential=MagicMock(), environment="dev") assert isinstance(result, DeploymentResult) assert result.responses is None class TestCollectResponses: """Test the _collect_responses helper function.""" def test_returns_none_when_responses_disabled(self): from fabric_cicd.publish import _collect_responses workspace = MagicMock() workspace.responses = {"Notebook": {"nb1": {"body": {}}}} assert _collect_responses(workspace, responses_enabled=False) is None def test_returns_none_when_workspace_is_none(self): from fabric_cicd.publish import _collect_responses assert _collect_responses(None, responses_enabled=True) is None def test_returns_none_when_both_responses_empty_dict(self): from fabric_cicd.publish import _collect_responses workspace = MagicMock() workspace.responses = {} workspace.unpublish_responses = {} assert _collect_responses(workspace, responses_enabled=True) is None def test_returns_none_when_both_responses_none(self): from fabric_cicd.publish import _collect_responses workspace = MagicMock() workspace.responses = None workspace.unpublish_responses = None assert _collect_responses(workspace, responses_enabled=True) is None def test_returns_none_when_responses_empty_and_unpublish_none(self): from fabric_cicd.publish import _collect_responses workspace = MagicMock() workspace.responses = {} workspace.unpublish_responses = None assert _collect_responses(workspace, responses_enabled=True) is None def test_returns_none_when_responses_none_and_unpublish_empty(self): from fabric_cicd.publish import _collect_responses workspace = MagicMock() workspace.responses = None workspace.unpublish_responses = {} assert _collect_responses(workspace, responses_enabled=True) is None def test_returns_publish_only_when_publish_available(self): from fabric_cicd.publish import _collect_responses workspace = MagicMock() workspace.responses = {"Notebook": {"nb1": {"body": {"id": "123"}}}} workspace.unpublish_responses = None result = _collect_responses(workspace, responses_enabled=True) assert result == {"publish": {"Notebook": {"nb1": {"body": {"id": "123"}}}}} def test_returns_unpublish_only_when_unpublish_available(self): from fabric_cicd.publish import _collect_responses workspace = MagicMock() workspace.responses = None workspace.unpublish_responses = {"Notebook": {"nb1": {"body": {"id": "456"}}}} result = _collect_responses(workspace, responses_enabled=True) assert result == {"unpublish": {"Notebook": {"nb1": {"body": {"id": "456"}}}}} def test_returns_both_when_both_available(self): from fabric_cicd.publish import _collect_responses workspace = MagicMock() workspace.responses = {"Notebook": {"nb1": {"body": {"id": "123"}}}} workspace.unpublish_responses = {"Notebook": {"nb2": {"body": {"id": "456"}}}} result = _collect_responses(workspace, responses_enabled=True) assert result == { "publish": {"Notebook": {"nb1": {"body": {"id": "123"}}}}, "unpublish": {"Notebook": {"nb2": {"body": {"id": "456"}}}}, } ================================================ FILE: tests/test_environment_publish.py ================================================ from pathlib import Path from typing import ClassVar import yaml from fabric_cicd._items import _environment as env_module class DummyFile: def __init__(self, file_path): # accept Path or string self.file_path = Path(file_path) # keep attributes other code may inspect self.relative_path = str(self.file_path).replace("\\", "/") self.type = "text" self.base64_payload = "payload" # Read contents from file if it exists, otherwise empty string if self.file_path.exists(): self.contents = self.file_path.read_text(encoding="utf-8") else: self.contents = "" class DummyItem: def __init__(self, name, file_paths): self.name = name self.item_files = [DummyFile(p) for p in file_paths] # Set path to the environment item directory (parent of the folder that # contains the file), e.g. if file is /tmp/Env/Setting/Sparkcompute.yml -> # path should be /tmp/Env if file_paths: p = Path(file_paths[0]) # if parent has at least one parent, choose grandparent; otherwise use parent self.path = p.parent.parent if p.parent.parent != Path() else p.parent else: self.path = Path() self.guid = None self.skip_publish = False # ---------- func_process_file tests ---------- def test_process_environment_file_non_sparkcompute(tmp_path): """Non-Sparkcompute files are returned unchanged.""" f = tmp_path / "EnvA" / "Libraries" / "lib.txt" f.parent.mkdir(parents=True, exist_ok=True) f.write_text("original content", encoding="utf-8") dummy = DummyFile(f) result = env_module._process_environment_file(None, DummyItem("EnvA", [f]), dummy) assert result == "original content" def test_process_environment_file_no_instance_pool(tmp_path): """Sparkcompute.yml without instance_pool_id is returned as re-serialized YAML.""" env_dir = tmp_path / "EnvB" setting_dir = env_dir / "Setting" setting_dir.mkdir(parents=True, exist_ok=True) sc = setting_dir / "Sparkcompute.yml" sc.write_text("driver_cores: 8\ndriver_memory: 56g\n", encoding="utf-8") dummy = DummyFile(sc) result = env_module._process_environment_file(None, DummyItem("EnvB", [sc]), dummy) parsed = yaml.safe_load(result) assert parsed["driver_cores"] == 8 assert parsed["driver_memory"] == "56g" assert "instance_pool_id" not in parsed def test_process_environment_file_replaces_instance_pool(tmp_path): """Sparkcompute.yml with instance_pool_id is resolved to the target pool GUID via API.""" env_dir = tmp_path / "EnvC" setting_dir = env_dir / "Setting" setting_dir.mkdir(parents=True, exist_ok=True) sc = setting_dir / "Sparkcompute.yml" sc.write_text("instance_pool_id: pool-123\ndriver_cores: 4\n", encoding="utf-8") class FakeWS: environment = "DEV" environment_parameter: ClassVar[dict] = { "spark_pool": [ { "instance_pool_id": "pool-123", "replace_value": {"DEV": {"type": "Capacity", "name": "MyPool"}}, } ] } base_api_url = "https://api.example/v1/workspaces/ws-id" def _get_workspace_pools(self): return [ {"id": "resolved-guid-abc", "name": "MyPool", "type": "Capacity"}, {"id": "other-guid", "name": "OtherPool", "type": "Workspace"}, ] dummy = DummyFile(sc) result = env_module._process_environment_file(FakeWS(), DummyItem("EnvC", [sc]), dummy) parsed = yaml.safe_load(result) assert parsed["instance_pool_id"] == "resolved-guid-abc" assert "instance_pool" not in parsed assert parsed["driver_cores"] == 4 def test_process_environment_file_pool_with_item_name_filter(tmp_path): """instance_pool_id replacement respects the optional item_name filter.""" env_dir = tmp_path / "EnvD" setting_dir = env_dir / "Setting" setting_dir.mkdir(parents=True, exist_ok=True) sc = setting_dir / "Sparkcompute.yml" sc.write_text("instance_pool_id: pool-456\n", encoding="utf-8") class FakeWS: environment = "PROD" environment_parameter: ClassVar[dict] = { "spark_pool": [ { "instance_pool_id": "pool-456", "replace_value": {"PROD": {"type": "Workspace", "name": "WsPool"}}, "item_name": "EnvD", } ] } base_api_url = "https://api.example/v1/workspaces/ws-id" def _get_workspace_pools(self): return [{"id": "ws-pool-guid", "name": "WsPool", "type": "Workspace"}] dummy = DummyFile(sc) result = env_module._process_environment_file(FakeWS(), DummyItem("EnvD", [sc]), dummy) parsed = yaml.safe_load(result) assert parsed["instance_pool_id"] == "ws-pool-guid" def test_process_environment_file_pool_no_match(tmp_path): """When no spark_pool entry matches, instance_pool_id is left as-is.""" env_dir = tmp_path / "EnvE" setting_dir = env_dir / "Setting" setting_dir.mkdir(parents=True, exist_ok=True) sc = setting_dir / "Sparkcompute.yml" sc.write_text("instance_pool_id: unmatched-pool\n", encoding="utf-8") class FakeWS: environment = "DEV" environment_parameter: ClassVar[dict] = { "spark_pool": [{"instance_pool_id": "different-pool", "replace_value": {"DEV": "something"}}] } base_api_url = "https://api.example/v1/workspaces/ws-id" def _get_workspace_pools(self): return [{"id": "guid-other", "name": "OtherPool", "type": "Capacity"}] dummy = DummyFile(sc) result = env_module._process_environment_file(FakeWS(), DummyItem("EnvE", [sc]), dummy) parsed = yaml.safe_load(result) assert "instance_pool_id" in parsed assert parsed["instance_pool_id"] == "unmatched-pool" def test_process_environment_file_no_spark_pool_param(tmp_path): """When environment_parameter has no spark_pool, instance_pool_id is left as-is.""" env_dir = tmp_path / "EnvF" setting_dir = env_dir / "Setting" setting_dir.mkdir(parents=True, exist_ok=True) sc = setting_dir / "Sparkcompute.yml" sc.write_text("instance_pool_id: some-pool\n", encoding="utf-8") class FakeWS: environment = "DEV" environment_parameter: ClassVar[dict] = {} base_api_url = "https://api.example/v1/workspaces/ws-id" def _get_workspace_pools(self): return [] dummy = DummyFile(sc) result = env_module._process_environment_file(FakeWS(), DummyItem("EnvF", [sc]), dummy) parsed = yaml.safe_load(result) assert parsed["instance_pool_id"] == "some-pool" def test_resolve_pool_id_success(): """_resolve_pool_id returns the matching pool GUID from the API response.""" pools = [ {"id": "guid-cap", "name": "CapPool", "type": "Capacity"}, {"id": "guid-ws", "name": "WsPool", "type": "Workspace"}, ] assert env_module._resolve_pool_id(pools, pool_name="CapPool", pool_type="Capacity") == "guid-cap" assert env_module._resolve_pool_id(pools, pool_name="WsPool", pool_type="Workspace") == "guid-ws" def test_resolve_pool_id_not_found(): """_resolve_pool_id raises when no matching pool is found.""" import pytest pools = [{"id": "guid-other", "name": "OtherPool", "type": "Capacity"}] with pytest.raises(Exception, match="Could not resolve custom Spark pool"): env_module._resolve_pool_id(pools, pool_name="MissingPool", pool_type="Workspace") # ---------- Publisher integration tests ---------- def test_publish_environments_passes_func_process_file(tmp_path): """ Ensure EnvironmentPublisher passes func_process_file to _publish_item and no longer passes shell_only_publish or exclude_path. """ captured = {} class FakeEndpoint: def invoke(self, *_args, **_kwargs): return {"body": {"value": []}} class FakeWorkspace: def __init__(self): p = tmp_path / "EnvX" / "Setting" / "Sparkcompute.yml" self.repository_items = {"Environment": {"EnvX": DummyItem("EnvX", [p])}} self.publish_item_name_exclude_regex = None self.publish_folder_path_exclude_regex = None self.items_to_include = None self.base_api_url = "https://example" self.endpoint = FakeEndpoint() self.repository_directory = tmp_path self.responses = None def _get_workspace_pools(self): return [] def _publish_item(self, item_name, item_type, **kwargs): captured["called_with"] = kwargs self.repository_items[item_type][item_name].skip_publish = True ws = FakeWorkspace() env_module.EnvironmentPublisher(ws).publish_all() assert "func_process_file" in captured["called_with"] assert captured["called_with"]["func_process_file"] is env_module._process_environment_file assert "shell_only_publish" not in captured["called_with"] assert "exclude_path" not in captured["called_with"] # ---------- End-to-end style tests ---------- def test_end_to_end_environment_setting_only(tmp_path): """ End-to-end style test for an Environment item that contains only Setting. Verifies: create item (POST /items) and submit publish (POST /staging/publish). Sparkcompute.yml is included in the regular item definition—no separate PATCH sparkcompute call is made. """ env_dir = tmp_path / "EnvEnd1" setting_dir = env_dir / "Setting" setting_dir.mkdir(parents=True, exist_ok=True) spark_yaml = setting_dir / "Sparkcompute.yml" spark_yaml.write_text("driver_cores: 4\n", encoding="utf-8") calls = [] class FakeEndpoint: def invoke(self, method=None, url=None, body=None, **_kwargs): calls.append((method, url, body)) if method == "GET" and url.endswith("/environments/"): return {"body": {"value": []}} if method == "POST" and url.endswith("/items"): return {"body": {"id": "guid-123"}} if method == "POST" and url.endswith("/staging/publish?beta=False"): return {"status": 202} return {} class FakeWorkspace: def __init__(self): self.repository_items = {"Environment": {"EnvEnd1": DummyItem("EnvEnd1", [spark_yaml])}} self.publish_item_name_exclude_regex = None self.publish_folder_path_exclude_regex = None self.items_to_include = None self.base_api_url = "https://api.example" self.endpoint = FakeEndpoint() self.repository_directory = tmp_path self.responses = None self.environment_parameter = {} def _get_workspace_pools(self): return [] def _publish_item(self, item_name, item_type, **_kwargs): item = self.repository_items[item_type][item_name] if not item.guid: resp = self.endpoint.invoke(method="POST", url=f"{self.base_api_url}/items", body={}) item.guid = resp["body"]["id"] ws = FakeWorkspace() env_module.EnvironmentPublisher(ws).publish_all() urls = [c[1] for c in calls] assert any("/items" in u and u.endswith("/items") for u in urls), "Create item call missing" assert any(u.endswith("/staging/publish?beta=False") for u in urls), "Publish submit missing" assert not any("sparkcompute" in u for u in urls), "Unexpected sparkcompute PATCH call" def test_end_to_end_environment_with_libraries(tmp_path): """ End-to-end style test for an Environment item that contains both Setting and Libraries. Verifies create/update flow and staging publish—no separate PATCH sparkcompute call. """ env_dir = tmp_path / "EnvEnd2" setting_dir = env_dir / "Setting" libs_dir = env_dir / "Libraries" setting_dir.mkdir(parents=True, exist_ok=True) libs_dir.mkdir(parents=True, exist_ok=True) spark_yaml = setting_dir / "Sparkcompute.yml" spark_yaml.write_text("driver_cores: 8\n", encoding="utf-8") (libs_dir / "lib.zip").write_text("dummy", encoding="utf-8") calls = [] class FakeEndpoint: def invoke(self, method=None, url=None, body=None, **_kwargs): calls.append((method, url, body)) if method == "GET" and url.endswith("/environments/"): return {"body": {"value": []}} if method == "POST" and url.endswith("/items"): return {"body": {"id": "guid-456"}} if method == "POST" and "updateDefinition" in url: return {"status": 200} if method == "POST" and url.endswith("/staging/publish?beta=False"): return {"status": 202} return {} class FakeWorkspace: def __init__(self): p_set = spark_yaml p_lib = libs_dir / "lib.zip" self.repository_items = {"Environment": {"EnvEnd2": DummyItem("EnvEnd2", [p_set, p_lib])}} self.publish_item_name_exclude_regex = None self.publish_folder_path_exclude_regex = None self.items_to_include = None self.base_api_url = "https://api.example" self.endpoint = FakeEndpoint() self.repository_directory = tmp_path self.responses = None self.environment_parameter = {} def _get_workspace_pools(self): return [] def _publish_item(self, item_name, item_type, **_kwargs): item = self.repository_items[item_type][item_name] if not item.guid: resp = self.endpoint.invoke(method="POST", url=f"{self.base_api_url}/items", body={}) item.guid = resp["body"]["id"] self.endpoint.invoke( method="POST", url=f"{self.base_api_url}/items/{item.guid}/updateDefinition?updateMetadata=True", body={}, ) ws = FakeWorkspace() env_module.EnvironmentPublisher(ws).publish_all() urls = [c[1] for c in calls] assert any("/items" in u and u.endswith("/items") for u in urls), "Create item call missing" assert any("updateDefinition" in u for u in urls), "updateDefinition call missing" assert any(u.endswith("/staging/publish?beta=False") for u in urls), "Publish submit missing" assert not any("sparkcompute" in u for u in urls), "Unexpected sparkcompute PATCH call" ================================================ FILE: tests/test_fabric_workspace.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. import json import re import tempfile from pathlib import Path from unittest.mock import MagicMock, patch import pytest import yaml from fixtures.credentials import DummyTokenCredential from fabric_cicd import configure_fabric_fqdn from fabric_cicd.fabric_workspace import FabricWorkspace, constants @pytest.fixture def mock_endpoint(): """Mock FabricEndpoint to avoid real API calls.""" mock = MagicMock() def mock_invoke(method, url, body=None, **_kwargs): if method == "POST" and url.endswith("/items"): return { "body": { "id": "mock-item-id-12345", "workspaceId": "mock-workspace-id", "displayName": body.get("displayName", "Test Item") if body else "Test Item", "type": body.get("type", "Unknown") if body else "Unknown", } } if method == "POST" and "updateDefinition" in url: return {"body": {"message": "Definition updated successfully"}} if method == "PATCH" and "items/" in url: return {"body": {"message": "Item metadata updated successfully"}} return {"body": {"value": []}} mock.invoke.side_effect = mock_invoke return mock @pytest.fixture def temp_workspace_dir(): """Create a temporary directory structure for testing.""" with tempfile.TemporaryDirectory() as temp_dir: yield Path(temp_dir) @pytest.fixture def valid_workspace_id(): """Return a valid workspace ID in GUID format.""" return "12345678-1234-5678-abcd-1234567890ab" @pytest.fixture def utf8_test_chars(): """Provide sample UTF-8 characters for testing.""" return {"nordic": "Ö æ ø", "european": "ñ é ü ß ç", "asian": "你好", "mixed": "ñ é ü ß ç 你好"} def create_parameter_file(dir_path, utf8_chars): """Create a parameter file with UTF-8 characters.""" parameter_file_path = dir_path / "parameter.yml" parameter_content = { "find_replace": [ { "find_value": f"Production {utf8_chars['mixed']}", "replace_value": { utf8_chars["nordic"]: "12345678-1234-5678-abcd-1234567890ab", utf8_chars["asian"]: "21345678-1234-5678-abcd-1234567890ab", }, } ] } with parameter_file_path.open("w", encoding="utf-8") as f: yaml.dump(parameter_content, f, allow_unicode=True) return parameter_content def create_platform_metadata(dir_path, utf8_chars): """Create a .platform metadata file with UTF-8 characters.""" item_dir = dir_path / "test_item" item_dir.mkdir(parents=True, exist_ok=True) platform_file_path = item_dir / ".platform" metadata_content = { "metadata": { "type": "Notebook", "displayName": f"Test Notebook with {utf8_chars['nordic']}", "description": f"Description with {utf8_chars['mixed']}", }, "config": {"logicalId": "test-logical-id"}, } with platform_file_path.open("w", encoding="utf-8") as f: json.dump(metadata_content, f, ensure_ascii=False) with (item_dir / "dummy.txt").open("w", encoding="utf-8") as f: f.write("Dummy file") return metadata_content @pytest.fixture def patched_fabric_workspace(mock_endpoint): """Return a factory function to create a patched FabricWorkspace.""" def _create_workspace(workspace_id, repository_directory, item_type_in_scope=None, **kwargs): fabric_endpoint_patch = patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint) refresh_items_patch = patch.object( FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", {}) ) refresh_folders_patch = patch.object( FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) ) with fabric_endpoint_patch, refresh_items_patch, refresh_folders_patch: workspace = FabricWorkspace( workspace_id=workspace_id, repository_directory=repository_directory, item_type_in_scope=item_type_in_scope, token_credential=DummyTokenCredential(), **kwargs, ) # Call refresh methods to populate workspace data workspace._refresh_deployed_folders() workspace._refresh_repository_folders() workspace._refresh_deployed_items() workspace._refresh_repository_items() return workspace return _create_workspace def test_parameter_file_with_utf8_chars( temp_workspace_dir, patched_fabric_workspace, valid_workspace_id, utf8_test_chars ): """Test that parameter file with UTF-8 characters is read correctly.""" create_parameter_file(temp_workspace_dir, utf8_test_chars) with patch.object(FabricWorkspace, "_refresh_repository_items"): workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), item_type_in_scope=["Environment"], ) key1 = f"Production {utf8_test_chars['mixed']}" key2 = utf8_test_chars["nordic"] key3 = utf8_test_chars["asian"] for param_dict in workspace.environment_parameter.get("find_replace"): assert key1 == param_dict["find_value"] assert key2 in param_dict["replace_value"] assert key3 in param_dict["replace_value"] def test_platform_metadata_with_utf8_chars( temp_workspace_dir, patched_fabric_workspace, valid_workspace_id, utf8_test_chars ): """Test that .platform metadata file with UTF-8 characters is read correctly.""" create_platform_metadata(temp_workspace_dir, utf8_test_chars) with patch.object(FabricWorkspace, "_refresh_parameter_file"): workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook"], ) item_name = f"Test Notebook with {utf8_test_chars['nordic']}" item = workspace.repository_items["Notebook"][item_name] assert "Notebook" in workspace.repository_items assert item_name in workspace.repository_items["Notebook"] assert item.name == item_name assert item.description == f"Description with {utf8_test_chars['mixed']}" def test_environment_param_with_utf8_chars( temp_workspace_dir, patched_fabric_workspace, valid_workspace_id, utf8_test_chars ): """Test that environment parameter with UTF-8 characters is preserved.""" with patch.object(FabricWorkspace, "_refresh_repository_items"): workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), item_type_in_scope=["Environment"], environment=utf8_test_chars["nordic"], ) assert workspace.environment == utf8_test_chars["nordic"] def test_workspace_id_replacement_in_json(patched_fabric_workspace, valid_workspace_id, temp_workspace_dir): """Test that workspace IDs are properly replaced in JSON files (like pipeline-content.json).""" # JSON content with workspace ID that should be replaced json_content = """{ "properties": { "activities": [ { "type": "TridentNotebook", "typeProperties": { "notebookId": "99b570c5-0c79-9dc4-4c9b-fa16c621384c", "workspaceId": "00000000-0000-0000-0000-000000000000" } } ] } }""" with patch.object(FabricWorkspace, "_refresh_repository_items"): workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), item_type_in_scope=["DataPipeline"], ) # Test the workspace ID replacement function result = workspace._replace_workspace_ids(json_content) # Verify that the default workspace ID was replaced with the target workspace ID assert "00000000-0000-0000-0000-000000000000" not in result assert valid_workspace_id in result assert '"workspaceId": "' + valid_workspace_id + '"' in result def test_workspace_id_replacement_in_python(patched_fabric_workspace, valid_workspace_id, temp_workspace_dir): """Test that workspace IDs are properly replaced in Python files (like notebook-content.py).""" # Python content with workspace ID that should be replaced (as in notebook metadata) python_content = """# META { # META "dependencies": { # META "environment": { # META "environmentId": "a277ea4a-e87f-8537-4ce0-39db11d4aade", # META "workspaceId": "00000000-0000-0000-0000-000000000000" # META } # META } # META }""" with patch.object(FabricWorkspace, "_refresh_repository_items"): workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook"], ) # Test the workspace ID replacement function result = workspace._replace_workspace_ids(python_content) # Verify that the default workspace ID was replaced with the target workspace ID assert "00000000-0000-0000-0000-000000000000" not in result assert valid_workspace_id in result assert 'workspaceId": "' + valid_workspace_id + '"' in result def test_workspace_id_replacement_eventstream_json(patched_fabric_workspace, valid_workspace_id, temp_workspace_dir): """Test workspace ID replacement in Eventstream JSON files with multiple occurrences.""" eventstream_content = """{ "destinations": [ { "name": "DataActivator", "type": "Activator", "properties": { "workspaceId": "00000000-0000-0000-0000-000000000000", "itemId": "c3bf82de-14b6-af39-4852-dda67eccd7c0" } }, { "name": "Lakehouse", "type": "Lakehouse", "properties": { "workspaceId": "00000000-0000-0000-0000-000000000000", "itemId": "c916eeb0-dd6a-ae32-4f4f-966d2414b239" } }, { "name": "Eventhouse", "type": "Eventhouse", "properties": { "workspaceId": "00000000-0000-0000-0000-000000000000", "itemId": "a51e98dd-5993-8e1c-443f-02aa53d4db74" } } ] }""" with patch.object(FabricWorkspace, "_refresh_repository_items"): workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), item_type_in_scope=["Eventstream"], ) result = workspace._replace_workspace_ids(eventstream_content) # Verify all three workspace IDs were replaced assert "00000000-0000-0000-0000-000000000000" not in result assert result.count(f'"workspaceId": "{valid_workspace_id}"') == 3 def test_workspace_id_replacement_yaml_format(patched_fabric_workspace, valid_workspace_id, temp_workspace_dir): """Test workspace ID replacement in YAML-style formats.""" yaml_content = """ configuration: lakehouse: default_lakehouse_workspace_id: "00000000-0000-0000-0000-000000000000" environment: workspaceId = "00000000-0000-0000-0000-000000000000" other: workspace: "00000000-0000-0000-0000-000000000000" """ with patch.object(FabricWorkspace, "_refresh_repository_items"): workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), item_type_in_scope=["Environment"], ) result = workspace._replace_workspace_ids(yaml_content) # Verify all different property name formats are replaced assert "00000000-0000-0000-0000-000000000000" not in result assert f'default_lakehouse_workspace_id: "{valid_workspace_id}"' in result assert f'workspaceId = "{valid_workspace_id}"' in result assert f'workspace: "{valid_workspace_id}"' in result def test_workspace_id_replacement_mixed_formats(patched_fabric_workspace, valid_workspace_id, temp_workspace_dir): """Test workspace ID replacement with mixed JSON and YAML formats in same content.""" mixed_content = """{ "pipeline": { "properties": { "workspaceId": "00000000-0000-0000-0000-000000000000" } }, "configuration": { "default_lakehouse_workspace_id": "00000000-0000-0000-0000-000000000000", "workspace" = "00000000-0000-0000-0000-000000000000" } }""" with patch.object(FabricWorkspace, "_refresh_repository_items"): workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), item_type_in_scope=["DataPipeline"], ) result = workspace._replace_workspace_ids(mixed_content) # Verify all formats are replaced correctly assert "00000000-0000-0000-0000-000000000000" not in result assert f'"workspaceId": "{valid_workspace_id}"' in result assert f'"default_lakehouse_workspace_id": "{valid_workspace_id}"' in result assert f'"workspace" = "{valid_workspace_id}"' in result def test_workspace_id_replacement_whitespace_variations( patched_fabric_workspace, valid_workspace_id, temp_workspace_dir ): """Test workspace ID replacement with various whitespace patterns.""" whitespace_content = """ { "test1": { "workspaceId":"00000000-0000-0000-0000-000000000000" }, "test2": { "workspaceId" : "00000000-0000-0000-0000-000000000000" }, "test3": { workspaceId = "00000000-0000-0000-0000-000000000000" }, "test4": { "workspace" : "00000000-0000-0000-0000-000000000000" } } """ with patch.object(FabricWorkspace, "_refresh_repository_items"): workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), item_type_in_scope=["DataPipeline"], ) result = workspace._replace_workspace_ids(whitespace_content) # Verify all whitespace variations are handled assert "00000000-0000-0000-0000-000000000000" not in result assert result.count(valid_workspace_id) == 4 def test_workspace_id_replacement_non_default_values_preserved( patched_fabric_workspace, valid_workspace_id, temp_workspace_dir ): """Test that non-default workspace IDs are NOT replaced (regression test).""" # Use a different workspace ID that should not be replaced other_workspace_id = "12345678-1234-1234-1234-123456789012" content_with_other_id = f'''{{ "properties": {{ "activities": [ {{ "type": "TridentNotebook", "typeProperties": {{ "workspaceId": "{other_workspace_id}", "notebookId": "99b570c5-0c79-9dc4-4c9b-fa16c621384c" }} }}, {{ "type": "TridentNotebook", "typeProperties": {{ "workspaceId": "00000000-0000-0000-0000-000000000000", "notebookId": "88a570c5-0c79-9dc4-4c9b-fa16c621384c" }} }} ] }} }}''' with patch.object(FabricWorkspace, "_refresh_repository_items"): workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), item_type_in_scope=["DataPipeline"], ) result = workspace._replace_workspace_ids(content_with_other_id) # Verify only default workspace ID was replaced, other ID preserved assert "00000000-0000-0000-0000-000000000000" not in result assert other_workspace_id in result # This should be preserved assert result.count(valid_workspace_id) == 1 # Only one replacement assert result.count(other_workspace_id) == 1 # Original preserved def test_workspace_id_replacement_edge_cases(patched_fabric_workspace, valid_workspace_id, temp_workspace_dir): """Test workspace ID replacement edge cases and current regex behavior.""" edge_cases_content = """ // Comment with workspaceId: "00000000-0000-0000-0000-000000000000" - this gets replaced due to current regex { "validCase1": { "workspaceId": "00000000-0000-0000-0000-000000000000" }, "validCase2": { "default_lakehouse_workspace_id": "00000000-0000-0000-0000-000000000000" }, "invalidCase1": { "workspaceIdNot": "00000000-0000-0000-0000-000000000000" }, "invalidCase2": { "notworkspaceId": "00000000-0000-0000-0000-000000000000" }, "validCase3": { workspace: "00000000-0000-0000-0000-000000000000" } } """ with patch.object(FabricWorkspace, "_refresh_repository_items"): workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), item_type_in_scope=["DataPipeline"], ) result = workspace._replace_workspace_ids(edge_cases_content) # Current regex behavior: matches comments and partial matches like "notworkspaceId" # This documents the current behavior for regression testing assert result.count(valid_workspace_id) == 5 # comment, validCase1, validCase2, invalidCase2, validCase3 assert '"workspaceIdNot": "00000000-0000-0000-0000-000000000000"' in result # Should not be replaced (prefix case) assert f'"notworkspaceId": "{valid_workspace_id}"' in result # Gets replaced (suffix matches workspaceId) assert f'// Comment with workspaceId: "{valid_workspace_id}"' in result # Comment gets replaced def test_workspace_id_replacement_comprehensive_item_types( patched_fabric_workspace, valid_workspace_id, temp_workspace_dir ): """Test workspace ID replacement across different item type contexts.""" # Test content that might appear in different item types comprehensive_content = """ { "notebook": { "metadata": { "environment": { "workspaceId": "00000000-0000-0000-0000-000000000000" } } }, "pipeline": { "activities": [{ "typeProperties": { "workspaceId": "00000000-0000-0000-0000-000000000000" } }] }, "eventstream": { "destinations": [{ "properties": { "workspaceId": "00000000-0000-0000-0000-000000000000" } }] }, "lakehouse": { "default_lakehouse_workspace_id": "00000000-0000-0000-0000-000000000000" }, "environment": { "workspace": "00000000-0000-0000-0000-000000000000" } } """ # Test with different item types to ensure the replacement works regardless of item type context item_types_to_test = ["Notebook", "DataPipeline", "Eventstream", "Lakehouse", "Environment"] for item_type in item_types_to_test: with patch.object(FabricWorkspace, "_refresh_repository_items"): workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), item_type_in_scope=[item_type], ) result = workspace._replace_workspace_ids(comprehensive_content) # Verify all workspace IDs are replaced regardless of item type context assert "00000000-0000-0000-0000-000000000000" not in result, f"Failed for item type: {item_type}" assert result.count(valid_workspace_id) == 5, f"Incorrect replacement count for item type: {item_type}" def test_environment_parameter_replacement_issue(patched_fabric_workspace, temp_workspace_dir, valid_workspace_id): """Test that parameter replacement works correctly with different environment values. This test ensures that the issue where parameter replacement doesn't work when environment defaults to 'N/A' is properly handled. """ # Create parameter.yml file with environment-specific replacements parameter_content = """ find_replace: - find_value: "test-guid-to-replace" replace_value: PPE: "ppe-replacement-value" PROD: "prod-replacement-value" item_type: "Notebook" item_name: ["Test Notebook"] """ # Create notebook structure notebook_dir = temp_workspace_dir / "Test Notebook.Notebook" notebook_dir.mkdir(parents=True) notebook_content = 'test_value = "test-guid-to-replace"' # Write files (temp_workspace_dir / "parameter.yml").write_text(parameter_content) (notebook_dir / "notebook-content.py").write_text(notebook_content) from fabric_cicd._common._file import File from fabric_cicd._common._item import Item # Test 1: Without environment parameter (defaults to 'N/A') with patch.object(FabricWorkspace, "_refresh_repository_items"): workspace_no_env = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook"], ) # Test 2: With environment parameter (PPE) with patch.object(FabricWorkspace, "_refresh_repository_items"): workspace_with_env = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook"], environment="PPE", ) # Create test objects for parameter replacement test_item = Item(type="Notebook", name="Test Notebook", description="", guid="test-guid", path=notebook_dir) test_file = File(item_path=notebook_dir, file_path=notebook_dir / "notebook-content.py") # Test parameter replacement with default environment replaced_content_no_env = workspace_no_env._replace_parameters(test_file, test_item) # Test parameter replacement with specific environment replaced_content_with_env = workspace_with_env._replace_parameters(test_file, test_item) # Assertions # With default environment ('N/A'), replacement should NOT occur assert "test-guid-to-replace" in replaced_content_no_env, "Original value should remain when environment is N/A" assert "ppe-replacement-value" not in replaced_content_no_env, ( "Replacement should not occur with default environment" ) # With specific environment (PPE), replacement SHOULD occur assert "test-guid-to-replace" not in replaced_content_with_env, ( "Original value should be replaced when environment matches" ) assert "ppe-replacement-value" in replaced_content_with_env, "Replacement should occur with matching environment" def test_empty_logical_id_validation(temp_workspace_dir, patched_fabric_workspace, valid_workspace_id): """Test that empty logical IDs raise a ParsingError during repository refresh.""" from fabric_cicd._common._exceptions import ParsingError # Create a .platform file with empty logical ID item_dir = temp_workspace_dir / "TestItem.Notebook" item_dir.mkdir(parents=True, exist_ok=True) platform_file_path = item_dir / ".platform" metadata_content = { "metadata": { "type": "Notebook", "displayName": "Test Item with Empty Logical ID", "description": "Test item for empty logical ID validation", }, "config": {"logicalId": ""}, # Empty logical ID } with platform_file_path.open("w", encoding="utf-8") as f: json.dump(metadata_content, f, ensure_ascii=False) # Create a dummy content file with (item_dir / "dummy.txt").open("w", encoding="utf-8") as f: f.write("Dummy file content") # Test that ParsingError is raised when trying to refresh repository items with pytest.raises(ParsingError) as exc_info: patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook"], ) # Verify the error message contains the expected information assert "logicalId cannot be empty" in str(exc_info.value) assert str(platform_file_path) in str(exc_info.value) def test_whitespace_only_logical_id_validation(temp_workspace_dir, patched_fabric_workspace, valid_workspace_id): """Test that logical IDs with only whitespace raise a ParsingError.""" from fabric_cicd._common._exceptions import ParsingError # Create a .platform file with whitespace-only logical ID item_dir = temp_workspace_dir / "TestItem.Notebook" item_dir.mkdir(parents=True, exist_ok=True) platform_file_path = item_dir / ".platform" metadata_content = { "metadata": { "type": "Notebook", "displayName": "Test Item with Whitespace Logical ID", "description": "Test item for whitespace logical ID validation", }, "config": {"logicalId": " "}, # Whitespace-only logical ID } with platform_file_path.open("w", encoding="utf-8") as f: json.dump(metadata_content, f, ensure_ascii=False) # Create a dummy content file with (item_dir / "dummy.txt").open("w", encoding="utf-8") as f: f.write("Dummy file content") # Test that ParsingError is raised when trying to refresh repository items with pytest.raises(ParsingError) as exc_info: patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook"], ) # Verify the error message assert "logicalId cannot be empty" in str(exc_info.value) def test_valid_logical_id_works_correctly(temp_workspace_dir, patched_fabric_workspace, valid_workspace_id): """Test that valid logical IDs continue to work correctly after adding validation.""" # Create a .platform file with valid logical ID item_dir = temp_workspace_dir / "TestItem.Notebook" item_dir.mkdir(parents=True, exist_ok=True) platform_file_path = item_dir / ".platform" metadata_content = { "metadata": { "type": "Notebook", "displayName": "Test Item with Valid Logical ID", "description": "Test item for valid logical ID verification", }, "config": {"logicalId": "valid-logical-id-123"}, # Valid logical ID } with platform_file_path.open("w", encoding="utf-8") as f: json.dump(metadata_content, f, ensure_ascii=False) # Create a dummy content file with (item_dir / "dummy.txt").open("w", encoding="utf-8") as f: f.write("Dummy file content") # This should work without raising any exception workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook"] ) # Verify the item was loaded correctly (validation happens automatically during refresh) assert "Notebook" in workspace.repository_items assert "Test Item with Valid Logical ID" in workspace.repository_items["Notebook"] assert ( workspace.repository_items["Notebook"]["Test Item with Valid Logical ID"].logical_id == "valid-logical-id-123" ) def test_empty_logical_id_validation_during_publish(temp_workspace_dir, patched_fabric_workspace, valid_workspace_id): """Test that empty logical IDs are caught during workspace initialization.""" from fabric_cicd._common._exceptions import ParsingError # Create a .platform file with empty logical ID item_dir = temp_workspace_dir / "TestItem.Notebook" item_dir.mkdir(parents=True, exist_ok=True) platform_file_path = item_dir / ".platform" metadata_content = { "metadata": { "type": "Notebook", "displayName": "Test Item with Empty Logical ID", "description": "Test item for empty logical ID validation during publish", }, "config": {"logicalId": ""}, # Empty logical ID } with platform_file_path.open("w", encoding="utf-8") as f: json.dump(metadata_content, f, ensure_ascii=False) # Create a dummy content file with (item_dir / "dummy.txt").open("w", encoding="utf-8") as f: f.write("Dummy file content") # Test that ParsingError is raised during workspace initialization with pytest.raises(ParsingError) as exc_info: patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook"], ) # Verify the error message contains the expected information assert "logicalId cannot be empty" in str(exc_info.value) assert str(platform_file_path) in str(exc_info.value) def test_multiple_empty_logical_ids_validation(temp_workspace_dir, patched_fabric_workspace, valid_workspace_id): """Test that multiple empty logical IDs are all reported at once.""" from fabric_cicd._common._exceptions import ParsingError # Create multiple .platform files with empty logical IDs item_dirs = ["TestItem1.Notebook", "TestItem2.Notebook", "TestItem3.Environment"] platform_file_paths = [] for item_dir_name in item_dirs: item_dir = temp_workspace_dir / item_dir_name item_dir.mkdir(parents=True, exist_ok=True) platform_file_path = item_dir / ".platform" platform_file_paths.append(platform_file_path) item_type = "Notebook" if "Notebook" in item_dir_name else "Environment" metadata_content = { "metadata": { "type": item_type, "displayName": f"Test Item {item_dir_name}", "description": "Test item for multiple empty logical ID validation", }, "config": {"logicalId": ""}, # Empty logical ID } with platform_file_path.open("w", encoding="utf-8") as f: json.dump(metadata_content, f, ensure_ascii=False) # Create a dummy content file with (item_dir / "dummy.txt").open("w", encoding="utf-8") as f: f.write("Dummy file content") # Test that ParsingError is raised when trying to refresh repository items with pytest.raises(ParsingError) as exc_info: patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook", "Environment"], ) # Verify the error message contains information about all empty logical IDs error_message = str(exc_info.value) assert "logicalId cannot be empty in the following files:" in error_message for platform_file_path in platform_file_paths: assert str(platform_file_path) in error_message def test_single_empty_logical_id_validation_message(temp_workspace_dir, patched_fabric_workspace, valid_workspace_id): """Test that a single empty logical ID shows the original error format.""" from fabric_cicd._common._exceptions import ParsingError # Create a .platform file with empty logical ID item_dir = temp_workspace_dir / "TestItem.Notebook" item_dir.mkdir(parents=True, exist_ok=True) platform_file_path = item_dir / ".platform" metadata_content = { "metadata": { "type": "Notebook", "displayName": "Test Item with Empty Logical ID", "description": "Test item for single empty logical ID validation", }, "config": {"logicalId": ""}, # Empty logical ID } with platform_file_path.open("w", encoding="utf-8") as f: json.dump(metadata_content, f, ensure_ascii=False) # Create a dummy content file with (item_dir / "dummy.txt").open("w", encoding="utf-8") as f: f.write("Dummy file content") # Test that ParsingError is raised when trying to refresh repository items with pytest.raises(ParsingError) as exc_info: patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook"], ) # Verify the error message uses single file format (not "following files:") error_message = str(exc_info.value) assert "logicalId cannot be empty in " in error_message assert "following files:" not in error_message assert str(platform_file_path) in error_message def test_fabric_workspace_with_none_item_types_defaults_to_all( temp_workspace_dir, patched_fabric_workspace, valid_workspace_id ): """Test that FabricWorkspace works correctly when initialized with None item_type_in_scope (defaults to all available types).""" # Create a sample item to test with item_dir = temp_workspace_dir / "TestNotebook.Notebook" item_dir.mkdir(parents=True, exist_ok=True) platform_file_path = item_dir / ".platform" metadata_content = { "metadata": { "type": "Notebook", "displayName": "Test Notebook", "description": "Test notebook for None item types test", }, "config": {"logicalId": "test-logical-id-none"}, } with platform_file_path.open("w", encoding="utf-8") as f: json.dump(metadata_content, f, ensure_ascii=False) # Create a dummy content file with (item_dir / "dummy.txt").open("w", encoding="utf-8") as f: f.write("Dummy file content") # Test that workspace initializes correctly with None (default behavior) workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), # item_type_in_scope=None (default) ) # Verify that item_type_in_scope was expanded to all available types import fabric_cicd.constants as constants expected_types = list(constants.ACCEPTED_ITEM_TYPES) assert set(workspace.item_type_in_scope) == set(expected_types), ( f"Expected all item types, got {workspace.item_type_in_scope}" ) # Verify that the notebook item was loaded correctly assert "Notebook" in workspace.repository_items assert "Test Notebook" in workspace.repository_items["Notebook"] def test_parameter_file_path_types(temp_workspace_dir, patched_fabric_workspace, valid_workspace_id): """Test different path types for parameter_file_path in FabricWorkspace.""" # Absolute path - accepted param_file = temp_workspace_dir / "parameters.yml" param_file.write_text(""" find_replace: - find_value: "test-value" replace_value: DEV: "dev-replacement" """) workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), parameter_file_path=str(param_file), ) assert workspace.parameter_file_path == str(param_file) # Relative path - now resolved against repository directory but file doesn't exist # This should not raise an exception now, it's handled gracefully workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), parameter_file_path="relative/path/parameters.yml", ) # The workspace should be created successfully but with empty parameters assert workspace is not None assert hasattr(workspace, "environment_parameter") assert not workspace.environment_parameter def test_parameter_file_path_none(temp_workspace_dir, patched_fabric_workspace, valid_workspace_id): """Test None cases for parameter_file_path in FabricWorkspace.""" # Create a workspace with parameter_file_path set to None workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), parameter_file_path=None ) assert workspace.parameter_file_path is None # Create a workspace without parameter_file_path provided workspace = patched_fabric_workspace(workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir)) assert workspace.parameter_file_path is None def test_skip_parameterization_prevents_parameter_yml_auto_discovery( temp_workspace_dir, patched_fabric_workspace, valid_workspace_id ): """When skip_parameterization=True (config 'parameter' field absent), a parameter.yml present in the repository directory must NOT be loaded — _refresh_parameter_file must not be called at all, and environment_parameter must remain empty.""" # Create a parameter.yml in the repo — it must NOT be picked up param_file = temp_workspace_dir / "parameter.yml" param_file.write_text("find_replace:\n - find_value: 'secret'\n replace_value:\n DEV: 'replaced'\n") assert param_file.exists() workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), skip_parameterization=True, ) # environment_parameter must be empty — parameter.yml was not auto-discovered assert workspace.environment_parameter == {} def test_skip_parameterization_false_loads_explicit_parameter_file( temp_workspace_dir, patched_fabric_workspace, valid_workspace_id ): """When skip_parameterization=False (default) and an explicit parameter_file_path is given, the parameter file IS loaded and parameterization IS applied.""" param_file = temp_workspace_dir / "my_params.yml" param_file.write_text("find_replace:\n - find_value: 'secret'\n replace_value:\n DEV: 'replaced'\n") workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), parameter_file_path=str(param_file), environment="DEV", skip_parameterization=False, ) # Parameterization must be populated — the explicit file was loaded assert "find_replace" in workspace.environment_parameter def test_parameter_file_path_with_environment(temp_workspace_dir, patched_fabric_workspace, valid_workspace_id): """Test that FabricWorkspace works with both parameter_file_path and environment.""" param_file = temp_workspace_dir / "dev_parameters.yml" param_file.write_text(""" find_replace: - find_value: "test-value" replace_value: DEV: "dev-replacement" """) workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), parameter_file_path=str(param_file), environment="DEV", ) assert workspace.parameter_file_path == str(param_file) assert workspace.environment == "DEV" def test_parameter_file_path_backward_compatibility(temp_workspace_dir, patched_fabric_workspace, valid_workspace_id): """Test that existing code without parameter_file_path continues to work.""" workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook"], environment="PROD", ) # Should work as before without parameter_file_path assert workspace.parameter_file_path is None assert workspace.environment == "PROD" assert workspace.item_type_in_scope == ["Notebook"] def test_parameter_file_path_integration_with_parameter_class( temp_workspace_dir, patched_fabric_workspace, valid_workspace_id ): """Test that parameter_file_path integrates correctly with Parameter class.""" param_file = temp_workspace_dir / "test_parameters.yml" param_content = """ find_replace: - find_value: "test-value" replace_value: DEV: "dev-replacement" PROD: "prod-replacement" """ param_file.write_text(param_content) workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), parameter_file_path=str(param_file), environment="DEV", ) # The workspace should use the parameter file path assert workspace.parameter_file_path == str(param_file) # The parameter data should be loaded correctly # (Note: This is testing the integration, actual Parameter behavior tested separately) assert hasattr(workspace, "environment_parameter") assert "find_replace" in workspace.environment_parameter def test_parameter_file_path_invalid_type_rejected(temp_workspace_dir, patched_fabric_workspace, valid_workspace_id): """Test that FabricWorkspace handles invalid types for parameter_file_path.""" # This should not raise an exception now since Parameter handles the error internally workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), parameter_file_path=123, # Invalid type ) # The workspace should be created, but parameter loading should fail silently assert workspace is not None assert hasattr(workspace, "environment_parameter") # Environment parameter should be empty since the parameter file path was invalid assert not workspace.environment_parameter def test_no_token_credential_raises_error(temp_workspace_dir, valid_workspace_id): """Test that constructing FabricWorkspace without token_credential raises TypeError.""" # Create a simple platform file so directory validation passes notebook_dir = temp_workspace_dir / "Test Notebook" notebook_dir.mkdir() platform_file = notebook_dir / ".platform" platform_content = { "metadata": {"type": "Notebook", "displayName": "Test Notebook"}, "config": {"logicalId": "12345678-1234-5678-abcd-1234567890ab"}, } with platform_file.open("w", encoding="utf-8") as f: json.dump(platform_content, f) with pytest.raises(TypeError) as exc_info: FabricWorkspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), ) assert "token_credential" in str(exc_info.value) def test_base_api_url_kwarg_raises_error(temp_workspace_dir, valid_workspace_id): """Test that passing base_api_url as kwarg raises an error.""" from fabric_cicd._common._exceptions import InputError # Create a simple platform file notebook_dir = temp_workspace_dir / "Test Notebook" notebook_dir.mkdir() platform_file = notebook_dir / ".platform" platform_content = { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": {"type": "Notebook", "displayName": "Test Notebook"}, "config": {"version": "2.0", "logicalId": "12345678-1234-5678-abcd-1234567890ab"}, } with platform_file.open("w", encoding="utf-8") as f: json.dump(platform_content, f) # Test that base_api_url kwarg raises InputError with patch("fabric_cicd.fabric_workspace.FabricEndpoint"): with pytest.raises(InputError) as exc_info: FabricWorkspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), base_api_url="https://custom.api.url", token_credential=DummyTokenCredential(), ) # Verify the error message contains the expected text assert "base_api_url is no longer supported" in str(exc_info.value) assert "constants.DEFAULT_API_ROOT_URL" in str(exc_info.value) def test_resolve_workspace_name(patched_fabric_workspace, valid_workspace_id, temp_workspace_dir): """Tests _resolve_workspace_name resolves display name from workspace ID.""" mock_endpoint = MagicMock() mock_endpoint.invoke.return_value = { "body": { "id": "mock-workspace-id", "displayName": "My Workspace [DEV]", } } with patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint): workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), ) workspace.endpoint = mock_endpoint result = workspace._resolve_workspace_name() assert result == "My Workspace [DEV]" def test_resolve_workspace_name_not_found(patched_fabric_workspace, valid_workspace_id, temp_workspace_dir): """Tests _resolve_workspace_name raises InputError when displayName not in response.""" from fabric_cicd._common._exceptions import InputError mock_endpoint = MagicMock() mock_endpoint.invoke.return_value = {"body": {}} with patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint): workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), ) workspace.endpoint = mock_endpoint with pytest.raises(InputError, match="Workspace name could not be resolved from workspace ID"): workspace._resolve_workspace_name() def test_lookup_item_attribute(patched_fabric_workspace, valid_workspace_id, temp_workspace_dir): """Test that _lookup_item_attribute correctly finds items in another workspace.""" # Mock endpoint response for workspace items mock_endpoint = MagicMock() # Ensure the mock response exactly matches what's expected mock_response = { "body": { "value": [ {"id": "item-id-1234", "type": "Notebook", "displayName": "Test Notebook"}, {"id": "item-id-5678", "type": "DataPipeline", "displayName": "Test Pipeline"}, ] } } mock_endpoint.invoke.return_value = mock_response # Create a workspace with our mocked endpoint with patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint): workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook", "DataPipeline"], ) # Replace the endpoint attribute to ensure our mock is being used workspace.endpoint = mock_endpoint # Test finding an existing item item_id = workspace._lookup_item_attribute("target-workspace-id", "Notebook", "Test Notebook", "id") assert item_id == "item-id-1234" # Test API was called with correct parameters mock_endpoint.invoke.assert_called_with( method="GET", url=f"{constants.DEFAULT_API_ROOT_URL}/v1/workspaces/target-workspace-id/items" ) # Test finding a different item type item_id = workspace._lookup_item_attribute("target-workspace-id", "DataPipeline", "Test Pipeline", "id") assert item_id == "item-id-5678" # Test item not found - should raise InputError from fabric_cicd._common._exceptions import InputError with pytest.raises(InputError) as exc_info: workspace._lookup_item_attribute("target-workspace-id", "Notebook", "Non-Existent Notebook", "id") assert "Failed to look up item in workspace" in str(exc_info.value) assert "target-workspace-id" in str(exc_info.value) assert "Notebook" in str(exc_info.value) assert "Non-Existent Notebook" in str(exc_info.value) # Test item type not found - should raise InputError with pytest.raises(InputError) as exc_info: workspace._lookup_item_attribute("target-workspace-id", "NonExistentType", "Test Item", "id") assert "Failed to look up item in workspace" in str(exc_info.value) assert "target-workspace-id" in str(exc_info.value) assert "NonExistentType" in str(exc_info.value) assert "Test Item" in str(exc_info.value) def test_kqldatabase_folder_regex_root_eventhouse(): """KQLDatabase under top-level Eventhouse .children: group(1) is empty string.""" pattern = re.compile(constants.KQL_DATABASE_FOLDER_PATH_REGEX) relative_path = "/SampleEventhouse.Eventhouse/.children/TaxiDB.KQLDatabase" match = pattern.match(relative_path) assert match is not None, "Regex should match a top-level Eventhouse .children path" assert match.group(1) == "", "Expected empty string for group(1) when Eventhouse is at repository root" def test_kqldatabase_folder_regex_nested_subfolder(): """KQLDatabase nested under a subfolder before Eventhouse: group(1) captures the subfolder path.""" pattern = re.compile(constants.KQL_DATABASE_FOLDER_PATH_REGEX) relative_path = "/subfolder/EventhouseName.Eventhouse/.children/DB.KQLDatabase" match = pattern.match(relative_path) assert match is not None, "Regex should match nested Eventhouse .children path" assert match.group(1) == "/subfolder", "Expected '/subfolder' captured as the parent path" def test_kqldatabase_folder_regex_no_match_edge_case(): """Edge case: paths that do not follow the Eventhouse/.children pattern should not match.""" pattern = re.compile(constants.KQL_DATABASE_FOLDER_PATH_REGEX) # Missing '.Eventhouse/.children' sequence bad_paths = [ "/SomeFolder/TaxiDB.KQLDatabase", # no Eventhouse container "/Another.Eventhouse/TaxiDB.KQLDatabase", # missing '.children' "/prefix/.children/TaxiDB.KQLDatabase", # missing Eventhouse segment ] for p in bad_paths: assert pattern.match(p) is None, f"Regex should not match path: {p}" def test_get_item_attribute_caching_basic(patched_fabric_workspace, valid_workspace_id, temp_workspace_dir): """Test that _get_item_attribute caches results and returns expected values.""" mock_endpoint = MagicMock() # Mock response for Lakehouse sqlendpoint attribute mock_response = {"body": {"properties": {"sqlEndpointProperties": {"connectionString": "test-connection-string"}}}} mock_endpoint.invoke.return_value = mock_response # Create workspace with mocked endpoint with patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint): workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), ) workspace.endpoint = mock_endpoint # Test fetching an attribute result = workspace._get_item_attribute( workspace_id="test-workspace-id", item_type="Lakehouse", item_guid="test-item-guid", item_name="Test Lakehouse", attribute_name="sqlendpoint", ) # Verify the result is as expected assert result == "test-connection-string" # Verify API was called once assert mock_endpoint.invoke.call_count == 1 mock_endpoint.invoke.assert_called_with( method="GET", url=f"{constants.DEFAULT_API_ROOT_URL}/v1/workspaces/test-workspace-id/lakehouses/test-item-guid", ) def test_get_item_attribute_caching_prevents_api_call(patched_fabric_workspace, valid_workspace_id, temp_workspace_dir): """Test that fetching the same attribute again uses cache and doesn't make API call.""" mock_endpoint = MagicMock() # Mock response for Lakehouse sqlendpoint attribute mock_response = {"body": {"properties": {"sqlEndpointProperties": {"connectionString": "test-connection-string"}}}} mock_endpoint.invoke.return_value = mock_response # Create workspace with mocked endpoint with patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint): workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), ) workspace.endpoint = mock_endpoint # First call - should make API call result1 = workspace._get_item_attribute( workspace_id="test-workspace-id", item_type="Lakehouse", item_guid="test-item-guid", item_name="Test Lakehouse", attribute_name="sqlendpoint", ) # Second call with same parameters - should use cache result2 = workspace._get_item_attribute( workspace_id="test-workspace-id", item_type="Lakehouse", item_guid="test-item-guid", item_name="Test Lakehouse", attribute_name="sqlendpoint", ) # Verify results are the same assert result1 == result2 == "test-connection-string" # Verify API was called only once (cached on second call) assert mock_endpoint.invoke.call_count == 1 def test_get_item_attribute_different_cache_keys(patched_fabric_workspace, valid_workspace_id, temp_workspace_dir): """Test that different cache keys don't collide and each makes separate API calls.""" mock_endpoint = MagicMock() # Mock response for different item types def mock_invoke_side_effect(*args, **kwargs): url = kwargs.get("url", args[1] if len(args) > 1 else "") if "lakehouses" in url: return { "body": { "properties": { "sqlEndpointProperties": { "id": "endpoint-id-123", "connectionString": "lakehouse-connection-string", } } } } if "warehouses" in url: return {"body": {"properties": {"connectionString": "warehouse-connection-string"}}} if "eventhouses" in url: return {"body": {"properties": {"queryServiceUri": "eventhouse-query-uri"}}} return {"body": {}} mock_endpoint.invoke.side_effect = mock_invoke_side_effect # Create workspace with mocked endpoint with patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint): workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), ) workspace.endpoint = mock_endpoint # Test different combinations to ensure no cache collisions # Different item types lakehouse_result = workspace._get_item_attribute("ws1", "Lakehouse", "guid1", "name1", "sqlendpoint") warehouse_result = workspace._get_item_attribute("ws1", "Warehouse", "guid1", "name1", "sqlendpoint") eventhouse_result = workspace._get_item_attribute("ws1", "Eventhouse", "guid1", "name1", "queryserviceuri") # Different workspace IDs lakehouse_ws2_result = workspace._get_item_attribute("ws2", "Lakehouse", "guid1", "name1", "sqlendpoint") # Different item GUIDs lakehouse_guid2_result = workspace._get_item_attribute("ws1", "Lakehouse", "guid2", "name1", "sqlendpoint") # Different item names lakehouse_name2_result = workspace._get_item_attribute("ws1", "Lakehouse", "guid1", "name2", "sqlendpoint") # Different attributes lakehouse_sqlendpointid_result = workspace._get_item_attribute( "ws1", "Lakehouse", "guid1", "name1", "sqlendpointid" ) # Verify all results are different and correct assert lakehouse_result == "lakehouse-connection-string" assert warehouse_result == "warehouse-connection-string" assert eventhouse_result == "eventhouse-query-uri" assert lakehouse_ws2_result == "lakehouse-connection-string" # Same API response assert lakehouse_guid2_result == "lakehouse-connection-string" # Same API response assert lakehouse_name2_result == "lakehouse-connection-string" # Same API response # Mock the API to return different values for sqlendpointid def mock_invoke_side_effect_extended(*args, **kwargs): url = kwargs.get("url", args[1] if len(args) > 1 else "") if "lakehouses" in url: if "guid1" in url: return { "body": { "properties": { "sqlEndpointProperties": { "id": "endpoint-id-123", "connectionString": "lakehouse-connection-string", } } } } # guid2 return { "body": { "properties": { "sqlEndpointProperties": { "id": "endpoint-id-456", "connectionString": "lakehouse-connection-string-2", } } } } return {"body": {}} mock_endpoint.invoke.side_effect = mock_invoke_side_effect_extended # Fetch sqlendpointid for guid1 lakehouse_sqlendpointid_result = workspace._get_item_attribute( "ws1", "Lakehouse", "guid1", "name1", "sqlendpointid" ) assert lakehouse_sqlendpointid_result == "endpoint-id-123" # Verify API was called for each unique cache key # We expect 7 calls: 3 initial + 1 for ws2 + 1 for guid2 + 1 for name2 + 1 for sqlendpointid assert mock_endpoint.invoke.call_count == 7 def test_get_item_attribute_edge_cases(patched_fabric_workspace, valid_workspace_id, temp_workspace_dir): """Test edge cases for _get_item_attribute to ensure cache doesn't introduce regressions.""" mock_endpoint = MagicMock() mock_endpoint.invoke.return_value = {"body": {}} # Create workspace with mocked endpoint with patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint): workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), ) workspace.endpoint = mock_endpoint # Test empty item_guid - should return empty string without API call result = workspace._get_item_attribute("ws1", "Lakehouse", "", "name1", "sqlendpoint") assert result == "" assert mock_endpoint.invoke.call_count == 0 # No API call made # Test None item_guid - should return empty string without API call result = workspace._get_item_attribute("ws1", "Lakehouse", None, "name1", "sqlendpoint") assert result == "" assert mock_endpoint.invoke.call_count == 0 # No API call made # Test unsupported item type - should return empty string without API call result = workspace._get_item_attribute("ws1", "UnsupportedType", "guid1", "name1", "someattr") assert result == "" assert mock_endpoint.invoke.call_count == 0 # No API call made # Test unsupported attribute for supported item type - should return empty string without API call result = workspace._get_item_attribute("ws1", "Lakehouse", "guid1", "name1", "unsupportedattr") assert result == "" assert mock_endpoint.invoke.call_count == 0 # No API call made # Test valid call that results in empty attribute value - should raise InputError mock_endpoint.invoke.return_value = { "body": { "properties": { "sqlEndpointProperties": { "connectionString": "" # Empty value } } } } from fabric_cicd._common._exceptions import InputError with pytest.raises(InputError) as exc_info: workspace._get_item_attribute("ws1", "Lakehouse", "guid1", "name1", "sqlendpoint") assert "Attribute value not found" in str(exc_info.value) assert "Lakehouse" in str(exc_info.value) assert "name1" in str(exc_info.value) # Verify the error case was not cached with pytest.raises(InputError): workspace._get_item_attribute("ws1", "Lakehouse", "guid1", "name1", "sqlendpoint") # Should still be only 1 API call (cached error) assert mock_endpoint.invoke.call_count == 2 def test_multiple_items_with_default_guid_logical_id(temp_workspace_dir, patched_fabric_workspace, valid_workspace_id): """Test that multiple items with DEFAULT_GUID as logical ID don't raise a duplicate error.""" # Create multiple items all using the default GUID (export API scenario) default_guid = constants.DEFAULT_GUID for i, item_dir_name in enumerate(["Notebook1.Notebook", "Notebook2.Notebook", "Pipeline1.DataPipeline"]): item_dir = temp_workspace_dir / item_dir_name item_dir.mkdir(parents=True, exist_ok=True) item_type = "Notebook" if "Notebook" in item_dir_name else "DataPipeline" metadata_content = { "metadata": { "type": item_type, "displayName": f"Item {i}", "description": "", }, "config": {"logicalId": default_guid}, } with (item_dir / ".platform").open("w", encoding="utf-8") as f: json.dump(metadata_content, f, ensure_ascii=False) with (item_dir / "dummy.txt").open("w", encoding="utf-8") as f: f.write("Dummy file content") # Should NOT raise any error workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook", "DataPipeline"], ) # Verify all items were loaded assert "Notebook" in workspace.repository_items assert len(workspace.repository_items["Notebook"]) == 2 assert "DataPipeline" in workspace.repository_items assert len(workspace.repository_items["DataPipeline"]) == 1 def test_duplicate_non_default_logical_id_raises_error( temp_workspace_dir, patched_fabric_workspace, valid_workspace_id ): """Test that duplicate non-default logical IDs still raise an error.""" from fabric_cicd._common._exceptions import FailedPublishedItemStatusError duplicate_logical_id = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" for i, item_dir_name in enumerate(["Notebook1.Notebook", "Notebook2.Notebook"]): item_dir = temp_workspace_dir / item_dir_name item_dir.mkdir(parents=True, exist_ok=True) metadata_content = { "metadata": { "type": "Notebook", "displayName": f"Duplicate Item {i}", "description": "", }, "config": {"logicalId": duplicate_logical_id}, } with (item_dir / ".platform").open("w", encoding="utf-8") as f: json.dump(metadata_content, f, ensure_ascii=False) with (item_dir / "dummy.txt").open("w", encoding="utf-8") as f: f.write("Dummy file content") with pytest.raises(FailedPublishedItemStatusError) as exc_info: patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook"], ) assert "Duplicate logicalId" in str(exc_info.value) assert duplicate_logical_id in str(exc_info.value) def test_replace_logical_ids_skips_default_guid(temp_workspace_dir, patched_fabric_workspace, valid_workspace_id): """Test that _replace_logical_ids skips items with DEFAULT_GUID as their logical ID.""" from fabric_cicd._common._item import Item with patch.object(FabricWorkspace, "_refresh_repository_items"): workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook"], ) # Set up repository items with DEFAULT_GUID logical IDs workspace.repository_items = { "Notebook": { "Notebook1": Item( type="Notebook", name="Notebook1", description="", guid="actual-guid-1111", logical_id=constants.DEFAULT_GUID, ), "Notebook2": Item( type="Notebook", name="Notebook2", description="", guid="actual-guid-2222", logical_id=constants.DEFAULT_GUID, ), } } # File content containing the default GUID (e.g., as a workspace ID placeholder) raw_file = f'{{"workspaceId": "{constants.DEFAULT_GUID}"}}' result = workspace._replace_logical_ids(raw_file) # DEFAULT_GUID should NOT have been replaced by any item GUID assert constants.DEFAULT_GUID in result assert "actual-guid-1111" not in result assert "actual-guid-2222" not in result def test_replace_logical_ids_replaces_non_default_guid( temp_workspace_dir, patched_fabric_workspace, valid_workspace_id ): """Test that _replace_logical_ids still replaces non-default logical IDs correctly.""" from fabric_cicd._common._item import Item with patch.object(FabricWorkspace, "_refresh_repository_items"): workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook"], ) logical_id = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" item_guid = "11111111-2222-3333-4444-555555555555" workspace.repository_items = { "Notebook": { "MyNotebook": Item( type="Notebook", name="MyNotebook", description="", guid=item_guid, logical_id=logical_id, ), } } raw_file = f'{{"notebookId": "{logical_id}"}}' result = workspace._replace_logical_ids(raw_file) assert logical_id not in result assert item_guid in result def test_mix_of_default_and_non_default_logical_ids(temp_workspace_dir, patched_fabric_workspace, valid_workspace_id): """Test repository with a mix of DEFAULT_GUID and unique logical IDs.""" unique_logical_id = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" # Item 1: export API item with default GUID item_dir_1 = temp_workspace_dir / "ExportedNotebook.Notebook" item_dir_1.mkdir(parents=True, exist_ok=True) metadata_1 = { "metadata": {"type": "Notebook", "displayName": "Exported Notebook", "description": ""}, "config": {"logicalId": constants.DEFAULT_GUID}, } with (item_dir_1 / ".platform").open("w", encoding="utf-8") as f: json.dump(metadata_1, f) with (item_dir_1 / "dummy.txt").open("w", encoding="utf-8") as f: f.write("content") # Item 2: git integration item with unique logical ID item_dir_2 = temp_workspace_dir / "GitNotebook.Notebook" item_dir_2.mkdir(parents=True, exist_ok=True) metadata_2 = { "metadata": {"type": "Notebook", "displayName": "Git Notebook", "description": ""}, "config": {"logicalId": unique_logical_id}, } with (item_dir_2 / ".platform").open("w", encoding="utf-8") as f: json.dump(metadata_2, f) with (item_dir_2 / "dummy.txt").open("w", encoding="utf-8") as f: f.write("content") # Item 3: another export API item with default GUID item_dir_3 = temp_workspace_dir / "ExportedPipeline.DataPipeline" item_dir_3.mkdir(parents=True, exist_ok=True) metadata_3 = { "metadata": {"type": "DataPipeline", "displayName": "Exported Pipeline", "description": ""}, "config": {"logicalId": constants.DEFAULT_GUID}, } with (item_dir_3 / ".platform").open("w", encoding="utf-8") as f: json.dump(metadata_3, f) with (item_dir_3 / "dummy.txt").open("w", encoding="utf-8") as f: f.write("content") # Should NOT raise any error workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook", "DataPipeline"], ) assert len(workspace.repository_items["Notebook"]) == 2 assert len(workspace.repository_items["DataPipeline"]) == 1 assert workspace.repository_items["Notebook"]["Exported Notebook"].logical_id == constants.DEFAULT_GUID assert workspace.repository_items["Notebook"]["Git Notebook"].logical_id == unique_logical_id assert workspace.repository_items["DataPipeline"]["Exported Pipeline"].logical_id == constants.DEFAULT_GUID def test_publish_variable_library_only_calls_replace_parameters( temp_workspace_dir, patched_fabric_workspace, valid_workspace_id ): """Test that Variable Library items only have _replace_parameters called, not logical ID or workspace ID replacement.""" workspace = patched_fabric_workspace(valid_workspace_id, str(temp_workspace_dir)) mock_file = MagicMock() mock_file.relative_path = "valueSets/Default.json" mock_file.type = "text" mock_file.file_path = Path("valueSets/Default.json") mock_file.contents = '{"key": "value", "workspace": "00000000-0000-0000-0000-000000000000"}' mock_file.base64_payload = {"path": "valueSets/Default.json", "payloadType": "InlineBase64"} mock_item = MagicMock() mock_item.guid = None mock_item.folder_id = "" mock_item.folder_path = "" mock_item.description = "" mock_item.logical_id = "test-logical-id" mock_item.item_files = [mock_file] mock_item.skip_publish = False mock_item.type = "VariableLibrary" mock_item.name = "TestVars" workspace.repository_items = {"VariableLibrary": {"TestVars": mock_item}} workspace.deployed_items = {} with ( patch.object(workspace, "_replace_logical_ids", wraps=workspace._replace_logical_ids) as mock_logical, patch.object(workspace, "_replace_parameters", side_effect=lambda file, _: file.contents) as mock_params, patch.object(workspace, "_replace_workspace_ids", wraps=workspace._replace_workspace_ids) as mock_ws, ): workspace._publish_item(item_name="TestVars", item_type="VariableLibrary") # _replace_parameters should be called mock_params.assert_called_once() # _replace_logical_ids and _replace_workspace_ids should NOT be called mock_logical.assert_not_called() mock_ws.assert_not_called() def test_publish_non_variable_library_calls_all_replacements( temp_workspace_dir, patched_fabric_workspace, valid_workspace_id ): """Test that non-Variable Library items still go through the full replacement pipeline.""" workspace = patched_fabric_workspace(valid_workspace_id, str(temp_workspace_dir)) mock_file = MagicMock() mock_file.relative_path = "notebook-content.py" mock_file.type = "text" mock_file.file_path = Path("notebook-content.py") mock_file.contents = "print('hello')" mock_file.base64_payload = {"path": "notebook-content.py", "payloadType": "InlineBase64"} mock_item = MagicMock() mock_item.guid = None mock_item.folder_id = "" mock_item.folder_path = "" mock_item.description = "" mock_item.logical_id = "test-logical-id" mock_item.item_files = [mock_file] mock_item.skip_publish = False mock_item.type = "Notebook" mock_item.name = "TestNotebook" workspace.repository_items = {"Notebook": {"TestNotebook": mock_item}} workspace.deployed_items = {} with ( patch.object(workspace, "_replace_logical_ids", side_effect=lambda x: x) as mock_logical, patch.object(workspace, "_replace_parameters", side_effect=lambda file, _: file.contents) as mock_params, patch.object(workspace, "_replace_workspace_ids", side_effect=lambda x: x) as mock_ws, ): workspace._publish_item(item_name="TestNotebook", item_type="Notebook") # All three replacement methods should be called mock_logical.assert_called_once() mock_params.assert_called_once() mock_ws.assert_called_once() def test_api_root_url_snapshot_is_not_retargeted_by_second_configure_call( temp_workspace_dir, patched_fabric_workspace, monkeypatch ): """Test that a constructed FabricWorkspace retains its snapshotted URL even if configure_fabric_fqdn is called again for a different workspace.""" workspace_id_a = "f953f3da-c5f0-4e36-a644-c85933e35e2f" workspace_id_b = "a1b2c3d4-e5f6-7890-abcd-ef1234567890" # Reset globals to defaults before test monkeypatch.setattr(constants, "DEFAULT_API_ROOT_URL", "https://api.powerbi.com") monkeypatch.setattr(constants, "FABRIC_API_ROOT_URL", "https://api.fabric.microsoft.com") # Configure for workspace a and construct it configure_fabric_fqdn(workspace_id_a) expected_fqdn_a = constants.DEFAULT_API_ROOT_URL # snapshot what was set with patch.object(FabricWorkspace, "_refresh_repository_items"): workspace_a = patched_fabric_workspace( workspace_id=workspace_id_a, repository_directory=str(temp_workspace_dir), ) # Now configure for workspace b configure_fabric_fqdn(workspace_id_b) # workspace_a should still use fqdn_a, not fqdn_b assert workspace_a._api_root_url == expected_fqdn_a assert workspace_a.base_api_url.startswith(expected_fqdn_a) ================================================ FILE: tests/test_fqdn_workspace_id.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. import pytest import fabric_cicd import fabric_cicd.constants as constants from fabric_cicd import configure_fabric_fqdn from fabric_cicd._common._validate_env_vars import _get_fabric_fqdn_url class TestGetFabricFqdnUrl: """Tests for the _get_fabric_fqdn_url helper function.""" def test_produces_correct_fqdn_url(self): url = _get_fabric_fqdn_url("f953f3da-c5f0-4e36-a644-c85933e35e2f") assert url == "https://f953f3dac5f04e36a644c85933e35e2f.zf9.w.api.fabric.microsoft.com" def test_rejects_workspace_id_without_dashes(self): with pytest.raises(ValueError, match="valid GUID with dashes"): _get_fabric_fqdn_url("f953f3dac5f04e36a644c85933e35e2f") class TestConfigureFabricFqdn: """Tests for configure_fabric_fqdn.""" def test_globals_updated(self, monkeypatch): monkeypatch.setattr(constants, "FABRIC_API_ROOT_URL", "https://api.fabric.microsoft.com") monkeypatch.setattr(constants, "DEFAULT_API_ROOT_URL", "https://api.powerbi.com") configure_fabric_fqdn("f953f3da-c5f0-4e36-a644-c85933e35e2f") expected = "https://f953f3dac5f04e36a644c85933e35e2f.zf9.w.api.fabric.microsoft.com" assert expected == constants.FABRIC_API_ROOT_URL assert expected == constants.DEFAULT_API_ROOT_URL def test_overwrite_warning_on_second_call(self, monkeypatch, mocker): monkeypatch.setattr(constants, "FABRIC_API_ROOT_URL", "https://api.fabric.microsoft.com") monkeypatch.setattr(constants, "DEFAULT_API_ROOT_URL", "https://api.powerbi.com") mock_logger = mocker.Mock() monkeypatch.setattr(fabric_cicd, "logger", mock_logger) configure_fabric_fqdn("f953f3da-c5f0-4e36-a644-c85933e35e2f") mock_logger.warning.assert_not_called() configure_fabric_fqdn("f953f3da-c5f0-4e36-a644-c85933e35e2f") mock_logger.warning.assert_called_once() ================================================ FILE: tests/test_git_diff_utils.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Tests for git diff utilities: get_changed_items() and validate_git_compare_ref().""" from unittest.mock import patch import pytest import fabric_cicd._common._git_diff_utils as git_utils from fabric_cicd._common._exceptions import InputError from fabric_cicd._common._validate_input import validate_git_compare_ref # ============================================================================= # Tests for validate_git_compare_ref() # ============================================================================= class TestValidateGitCompareRef: def test_accepts_common_valid_refs(self): assert validate_git_compare_ref("HEAD~1") == "HEAD~1" assert validate_git_compare_ref("main") == "main" assert validate_git_compare_ref("feature/my_branch") == "feature/my_branch" assert validate_git_compare_ref("release/v1.2.3") == "release/v1.2.3" def test_rejects_empty_string(self): with pytest.raises(InputError): validate_git_compare_ref("") def test_rejects_whitespace_only(self): with pytest.raises(InputError): validate_git_compare_ref(" ") def test_rejects_dash_prefixed(self): with pytest.raises(InputError): validate_git_compare_ref("-n") with pytest.raises(InputError): validate_git_compare_ref("--help") def test_rejects_invalid_characters(self): with pytest.raises(InputError): validate_git_compare_ref("ref;rm -rf /") def test_rejects_shell_metacharacters(self): """Prevent shell injection via backticks, pipes, dollar signs, etc.""" for ref in ["$(whoami)", "`id`", "ref|cat /etc/passwd", "ref&echo bad", "ref\nnewline"]: with pytest.raises(InputError): validate_git_compare_ref(ref) def test_rejects_non_string_input(self): """Non-string types must be rejected.""" with pytest.raises(InputError): validate_git_compare_ref(123) with pytest.raises(InputError): validate_git_compare_ref(None) def test_accepts_advanced_git_ref_syntax(self): """Valid git ref syntax including caret, tilde, and reflog notation.""" assert validate_git_compare_ref("HEAD^") == "HEAD^" assert validate_git_compare_ref("HEAD~3") == "HEAD~3" assert validate_git_compare_ref("main@{1}") == "main@{1}" assert validate_git_compare_ref("origin/main") == "origin/main" assert validate_git_compare_ref("v2.0.0") == "v2.0.0" assert validate_git_compare_ref("abc123def") == "abc123def" # ============================================================================= # Tests for _resolve_git_diff_path() # ============================================================================= class TestResolveGitDiffPath: """Tests for path validation/resolution from git diff output.""" def test_rejects_absolute_paths(self, tmp_path): result = git_utils._resolve_git_diff_path("/etc/passwd", tmp_path, tmp_path) assert result is None def test_rejects_path_traversal(self, tmp_path): result = git_utils._resolve_git_diff_path("../../../etc/passwd", tmp_path, tmp_path) assert result is None def test_rejects_null_bytes(self, tmp_path): result = git_utils._resolve_git_diff_path("file\x00.txt", tmp_path, tmp_path) assert result is None def test_accepts_valid_relative_path(self, tmp_path): sub = tmp_path / "MyItem" sub.mkdir() result = git_utils._resolve_git_diff_path("MyItem/file.py", tmp_path, tmp_path) assert result is not None assert result.name == "file.py" def test_rejects_path_outside_repo_directory(self, tmp_path): """A file under git root but outside the repo subdirectory is rejected.""" repo_sub = tmp_path / "workspace" repo_sub.mkdir() result = git_utils._resolve_git_diff_path("other/file.py", tmp_path, repo_sub) assert result is None # ============================================================================= # Tests for _find_platform_item() # ============================================================================= class TestFindPlatformItem: """Tests for .platform file discovery and parsing.""" def test_finds_platform_in_same_directory(self, tmp_path): item_dir = tmp_path / "MyItem.Notebook" item_dir.mkdir() (item_dir / ".platform").write_text( '{"metadata": {"type": "Notebook", "displayName": "MyItem"}}', encoding="utf-8" ) file_path = item_dir / "notebook.py" file_path.touch() result = git_utils._find_platform_item(file_path, tmp_path) assert result == ("MyItem", "Notebook") def test_returns_none_when_no_platform_file(self, tmp_path): item_dir = tmp_path / "NoItem" item_dir.mkdir() file_path = item_dir / "file.py" file_path.touch() result = git_utils._find_platform_item(file_path, tmp_path) assert result is None def test_returns_none_for_malformed_platform_json(self, tmp_path): item_dir = tmp_path / "BadItem" item_dir.mkdir() (item_dir / ".platform").write_text("not valid json", encoding="utf-8") file_path = item_dir / "file.py" file_path.touch() result = git_utils._find_platform_item(file_path, tmp_path) assert result is None def test_returns_none_when_metadata_missing_type(self, tmp_path): item_dir = tmp_path / "NoType" item_dir.mkdir() (item_dir / ".platform").write_text( '{"metadata": {"displayName": "NoType"}}', encoding="utf-8", ) file_path = item_dir / "file.py" file_path.touch() result = git_utils._find_platform_item(file_path, tmp_path) assert result is None # ============================================================================= # Tests for get_changed_items() # ============================================================================= class TestGetChangedItems: """Tests for the public get_changed_items() utility function.""" def _make_git_diff_output(self, lines: list[str]) -> str: return "\n".join(lines) def test_returns_changed_items_from_git_diff(self, tmp_path): """Returns items detected as modified/added by git diff.""" # Set up a fake item directory with a .platform file item_dir = tmp_path / "MyNotebook.Notebook" item_dir.mkdir() platform = item_dir / ".platform" platform.write_text( '{"metadata": {"type": "Notebook", "displayName": "MyNotebook"}}', encoding="utf-8", ) changed_file = item_dir / "notebook.py" changed_file.write_text("print('hello')", encoding="utf-8") diff_output = self._make_git_diff_output(["M\tMyNotebook.Notebook/notebook.py"]) git_root_patch = "fabric_cicd._common._config_validator._find_git_root" with ( patch(git_root_patch, return_value=tmp_path), patch("subprocess.run") as mock_run, ): mock_run.return_value.stdout = diff_output mock_run.return_value.returncode = 0 result = git_utils.get_changed_items(tmp_path) assert result == ["MyNotebook.Notebook"] def test_returns_empty_list_when_no_changes(self, tmp_path): """Returns an empty list when git diff reports no changed files.""" git_root_patch = "fabric_cicd._common._config_validator._find_git_root" with ( patch(git_root_patch, return_value=tmp_path), patch("subprocess.run") as mock_run, ): mock_run.return_value.stdout = "" mock_run.return_value.returncode = 0 result = git_utils.get_changed_items(tmp_path) assert result == [] def test_returns_empty_list_when_git_root_not_found(self, tmp_path): """Returns an empty list and logs a warning when no git root is found.""" git_root_patch = "fabric_cicd._common._config_validator._find_git_root" with patch(git_root_patch, return_value=None): result = git_utils.get_changed_items(tmp_path) assert result == [] def test_returns_empty_list_when_git_diff_fails(self, tmp_path): """Returns an empty list and logs a warning when git diff fails.""" import subprocess git_root_patch = "fabric_cicd._common._config_validator._find_git_root" with ( patch(git_root_patch, return_value=tmp_path), patch("subprocess.run", side_effect=subprocess.CalledProcessError(1, "git", stderr="bad ref")), ): result = git_utils.get_changed_items(tmp_path) assert result == [] def test_uses_custom_git_compare_ref(self, tmp_path): """Passes the custom git_compare_ref to the underlying git command.""" git_root_patch = "fabric_cicd._common._config_validator._find_git_root" with ( patch(git_root_patch, return_value=tmp_path), patch("subprocess.run") as mock_run, ): mock_run.return_value.stdout = "" mock_run.return_value.returncode = 0 git_utils.get_changed_items(tmp_path, git_compare_ref="main") call_args = mock_run.call_args[0][0] assert call_args == ["git", "diff", "--name-status", "main"] def test_excludes_files_outside_repository_directory(self, tmp_path): """Files changed outside the configured repository_directory are ignored.""" outside_dir = tmp_path / "other_repo" / "SomeItem.Notebook" outside_dir.mkdir(parents=True) diff_output = self._make_git_diff_output(["M\tother_repo/SomeItem.Notebook/item.py"]) git_root_patch = "fabric_cicd._common._config_validator._find_git_root" with ( patch(git_root_patch, return_value=tmp_path), patch("subprocess.run") as mock_run, ): mock_run.return_value.stdout = diff_output mock_run.return_value.returncode = 0 # Use a subdirectory as the repository_directory so "other_repo" is out of scope repo_subdir = tmp_path / "my_workspace" repo_subdir.mkdir() result = git_utils.get_changed_items(repo_subdir) assert result == [] def test_deduplicates_multiple_files_in_same_item(self, tmp_path): """Multiple changed files in the same item should produce a single entry.""" item_dir = tmp_path / "MyNotebook.Notebook" item_dir.mkdir() (item_dir / ".platform").write_text( '{"metadata": {"type": "Notebook", "displayName": "MyNotebook"}}', encoding="utf-8", ) (item_dir / "file1.py").write_text("a", encoding="utf-8") (item_dir / "file2.py").write_text("b", encoding="utf-8") diff_output = self._make_git_diff_output([ "M\tMyNotebook.Notebook/file1.py", "M\tMyNotebook.Notebook/file2.py", ]) git_root_patch = "fabric_cicd._common._config_validator._find_git_root" with ( patch(git_root_patch, return_value=tmp_path), patch("subprocess.run") as mock_run, ): mock_run.return_value.stdout = diff_output mock_run.return_value.returncode = 0 result = git_utils.get_changed_items(tmp_path) assert result == ["MyNotebook.Notebook"] def test_handles_renamed_files(self, tmp_path): """Renamed files (R status) should be detected via the new path.""" item_dir = tmp_path / "Renamed.Notebook" item_dir.mkdir() (item_dir / ".platform").write_text( '{"metadata": {"type": "Notebook", "displayName": "Renamed"}}', encoding="utf-8", ) (item_dir / "new_name.py").write_text("x", encoding="utf-8") diff_output = self._make_git_diff_output(["R100\tOld.Notebook/old.py\tRenamed.Notebook/new_name.py"]) git_root_patch = "fabric_cicd._common._config_validator._find_git_root" with ( patch(git_root_patch, return_value=tmp_path), patch("subprocess.run") as mock_run, ): mock_run.return_value.stdout = diff_output mock_run.return_value.returncode = 0 result = git_utils.get_changed_items(tmp_path) assert result == ["Renamed.Notebook"] def test_returns_empty_list_on_timeout(self, tmp_path): """A git diff timeout should return an empty list gracefully.""" import subprocess git_root_patch = "fabric_cicd._common._config_validator._find_git_root" with ( patch(git_root_patch, return_value=tmp_path), patch("subprocess.run", side_effect=subprocess.TimeoutExpired("git", 30)), ): result = git_utils.get_changed_items(tmp_path) assert result == [] def test_multiple_distinct_items(self, tmp_path): """Changes across multiple items should all be returned.""" for name, item_type in [("NB1", "Notebook"), ("Pipeline1", "DataPipeline")]: item_dir = tmp_path / f"{name}.{item_type}" item_dir.mkdir() (item_dir / ".platform").write_text( f'{{"metadata": {{"type": "{item_type}", "displayName": "{name}"}}}}', encoding="utf-8", ) (item_dir / "file.py").write_text("content", encoding="utf-8") diff_output = self._make_git_diff_output([ "M\tNB1.Notebook/file.py", "A\tPipeline1.DataPipeline/file.py", ]) git_root_patch = "fabric_cicd._common._config_validator._find_git_root" with ( patch(git_root_patch, return_value=tmp_path), patch("subprocess.run") as mock_run, ): mock_run.return_value.stdout = diff_output mock_run.return_value.returncode = 0 result = git_utils.get_changed_items(tmp_path) assert sorted(result) == ["NB1.Notebook", "Pipeline1.DataPipeline"] def test_rejects_dangerous_git_compare_ref(self, tmp_path): """Passing an invalid git_compare_ref should raise InputError before running git.""" with pytest.raises(InputError): git_utils.get_changed_items(tmp_path, git_compare_ref="--exec=whoami") ================================================ FILE: tests/test_hard_delete.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Test enable_hard_delete feature flag functionality.""" import json import tempfile from pathlib import Path from unittest.mock import MagicMock, patch import pytest from fixtures.credentials import DummyTokenCredential import fabric_cicd.constants as constants from fabric_cicd.constants import FeatureFlag from fabric_cicd.fabric_workspace import FabricWorkspace @pytest.fixture def mock_endpoint(): """Mock FabricEndpoint to capture DELETE requests.""" mock = MagicMock() mock.delete_urls = [] def mock_invoke(method, url, **_kwargs): if method == "DELETE": mock.delete_urls.append(url) return {"body": {}, "header": {}, "status_code": 200} if method == "GET" and "workspaces" in url and not url.endswith("/items"): return {"body": {"value": [], "capacityId": "test-capacity"}} if method == "GET" and url.endswith("/items"): return {"body": {"value": []}} return {"body": {"value": []}} mock.invoke.side_effect = mock_invoke return mock @pytest.fixture def test_workspace(mock_endpoint): """Create a test workspace with a notebook item.""" with tempfile.TemporaryDirectory() as temp_dir: temp_path = Path(temp_dir) notebook_dir = temp_path / "TestNotebook.Notebook" notebook_dir.mkdir(parents=True, exist_ok=True) platform_file = notebook_dir / ".platform" platform_file.write_text( json.dumps({ "metadata": { "kernel_info": {"name": "synapse_pyspark"}, "language_info": {"name": "python"}, } }) ) notebook_file = notebook_dir / "notebook-content.py" notebook_file.write_text("# Test notebook content\nprint('Hello World')") with ( patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), patch.object( FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", {}) ), patch.object( FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) ), patch.object(FabricWorkspace, "_refresh_repository_items", new=lambda _: None), patch.object(FabricWorkspace, "_refresh_repository_folders", new=lambda _: None), ): workspace = FabricWorkspace( workspace_id="12345678-1234-5678-abcd-1234567890ab", repository_directory=str(temp_path), item_type_in_scope=["Notebook"], token_credential=DummyTokenCredential(), ) yield workspace @pytest.fixture(autouse=True) def _clear_feature_flags(): """Clear feature flags before and after each test to avoid state leakage.""" constants.FEATURE_FLAG.discard(FeatureFlag.ENABLE_HARD_DELETE.value) yield constants.FEATURE_FLAG.discard(FeatureFlag.ENABLE_HARD_DELETE.value) def test_unpublish_item_without_hard_delete_flag(test_workspace, mock_endpoint): """Test that _unpublish_item uses a plain DELETE URL when flag is not set.""" item_guid = "mock-guid-123" test_workspace.deployed_items = {"Notebook": {"TestNotebook": MagicMock(guid=item_guid)}} mock_endpoint.delete_urls.clear() test_workspace._unpublish_item(item_name="TestNotebook", item_type="Notebook") assert len(mock_endpoint.delete_urls) == 1 delete_url = mock_endpoint.delete_urls[0] assert delete_url == f"{test_workspace.base_api_url}/items/{item_guid}" assert "hardDelete=true" not in delete_url def test_unpublish_item_with_hard_delete_flag(test_workspace, mock_endpoint): """Test that _unpublish_item appends ?hardDelete=True when flag is set.""" item_guid = "mock-guid-456" test_workspace.deployed_items = {"Notebook": {"TestNotebook": MagicMock(guid=item_guid)}} constants.FEATURE_FLAG.add(FeatureFlag.ENABLE_HARD_DELETE.value) mock_endpoint.delete_urls.clear() test_workspace._unpublish_item(item_name="TestNotebook", item_type="Notebook") assert len(mock_endpoint.delete_urls) == 1 delete_url = mock_endpoint.delete_urls[0] assert delete_url == f"{test_workspace.base_api_url}/items/{item_guid}?hardDelete=true" def test_hard_delete_flag_via_append_feature_flag(test_workspace, mock_endpoint): """Test that enable_hard_delete works when set via append_feature_flag.""" from fabric_cicd import append_feature_flag item_guid = "mock-guid-789" test_workspace.deployed_items = {"Notebook": {"TestNotebook": MagicMock(guid=item_guid)}} append_feature_flag(FeatureFlag.ENABLE_HARD_DELETE.value) mock_endpoint.delete_urls.clear() test_workspace._unpublish_item(item_name="TestNotebook", item_type="Notebook") assert len(mock_endpoint.delete_urls) == 1 assert "hardDelete=true" in mock_endpoint.delete_urls[0] ================================================ FILE: tests/test_integration_publish.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Integration test for publish operations using mock Fabric API server.""" import gzip import importlib import os import shutil from pathlib import Path from urllib.parse import urlparse import pytest from fixtures.credentials import DummyTokenCredential from fixtures.mock_fabric_server import MOCK_SERVER_PORT, MockFabricServer import fabric_cicd import fabric_cicd._common._validate_env_vars as validate_env_vars import fabric_cicd.constants @pytest.fixture def allow_localhost_http_for_integration(monkeypatch: pytest.MonkeyPatch): """ Test-only override: allow http://localhost for mocked integration servers. """ real_validate = validate_env_vars.validate_env_var_api_url def _validate_api_url_test(env_var_name: str, default_value: str) -> str: value = os.environ.get(env_var_name, default_value) parsed = urlparse(value) host = (parsed.hostname or "").lower() if parsed.scheme == "http" and host in {"localhost", "127.0.0.1", "::1"}: return value.rstrip("/") return real_validate(env_var_name, default_value) monkeypatch.setattr(validate_env_vars, "validate_env_var_api_url", _validate_api_url_test) return @pytest.fixture def mock_fabric_api_server(allow_localhost_http_for_integration): # noqa: ARG001 """ Start mock Fabric API server for the test. Yields the server and sets environment variables for API URLs. """ tests_dir = Path(__file__).parent trace_file_gz = tests_dir / "fixtures" / MockFabricServer.HTTP_TRACE_FILE trace_file = trace_file_gz.with_suffix("") if not trace_file_gz.exists(): pytest.skip( "http_trace.json.gz not found - run devtools/debug_trace_deployment.py first to generate trace data" ) if trace_file.exists(): trace_file.unlink() with gzip.open(trace_file_gz, "rb") as f_in, trace_file.open("wb") as f_out: shutil.copyfileobj(f_in, f_out) server = MockFabricServer(trace_file, port=MOCK_SERVER_PORT) original_default_api = os.environ.get("DEFAULT_API_ROOT_URL") original_fabric_api = os.environ.get("FABRIC_API_ROOT_URL") original_retry_delay = os.environ.get("FABRIC_CICD_RETRY_DELAY_OVERRIDE_SECONDS") os.environ["DEFAULT_API_ROOT_URL"] = f"http://127.0.0.1:{MOCK_SERVER_PORT}" os.environ["FABRIC_API_ROOT_URL"] = f"http://127.0.0.1:{MOCK_SERVER_PORT}" os.environ["FABRIC_CICD_RETRY_DELAY_OVERRIDE_SECONDS"] = "0" # reload only after env is set and override fixture is active importlib.reload(fabric_cicd.constants) server.start() yield server server.stop() if original_default_api is not None: os.environ["DEFAULT_API_ROOT_URL"] = original_default_api else: os.environ.pop("DEFAULT_API_ROOT_URL", None) if original_fabric_api is not None: os.environ["FABRIC_API_ROOT_URL"] = original_fabric_api else: os.environ.pop("FABRIC_API_ROOT_URL", None) if original_retry_delay is not None: os.environ["FABRIC_CICD_RETRY_DELAY_OVERRIDE_SECONDS"] = original_retry_delay else: os.environ.pop("FABRIC_CICD_RETRY_DELAY_OVERRIDE_SECONDS", None) importlib.reload(fabric_cicd.constants) def test_publish_all_items_integration(mock_fabric_api_server): # noqa: ARG001 """Test full publish_all_items workflow using mocked API responses.""" workspace_id = "00000000-0000-0000-0000-000000000000" environment_key = "PPE" root_directory = Path(__file__).resolve().parent.parent artifacts_folder = root_directory / "sample" / "workspace" item_types_to_deploy = [ "Dataflow", "DataPipeline", "Environment", "Eventhouse", "Eventstream", "KQLDatabase", "KQLQueryset", "Lakehouse", "MirroredDatabase", "MLExperiment", "Notebook", "Ontology", "Reflex", "Report", "SemanticModel", "SparkJobDefinition", "SQLDatabase", "VariableLibrary", "Warehouse", ] token_credential = DummyTokenCredential() for flag in ["enable_shortcut_publish", "continue_on_shortcut_failure"]: fabric_cicd.append_feature_flag(flag) target_workspace = fabric_cicd.FabricWorkspace( workspace_id=workspace_id, environment=environment_key, repository_directory=str(artifacts_folder), item_type_in_scope=item_types_to_deploy, token_credential=token_credential, ) fabric_cicd.publish_all_items(target_workspace) assert True, "Publish completed successfully" ================================================ FILE: tests/test_logging.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Tests for the logging module and wrapper functions.""" import logging import shutil import tempfile from logging.handlers import RotatingFileHandler from pathlib import Path from unittest.mock import patch import pytest from fabric_cicd import ( append_feature_flag, change_log_level, configure_external_file_logging, constants, disable_file_logging, ) from fabric_cicd._common._logging import ( CustomFormatter, PackageFilter, _build_console_message, _build_file_message, _cleanup_managed_handlers, _configure_console_handler, _configure_default_file_handler, _configure_external_file_handler, _mark_external_handler, _mark_handler, configure_logger, exception_handler, get_file_handler, log_header, ) def _close_all_file_handlers(): """Close all file handlers to release file locks on Windows.""" for logger_name in ("", "fabric_cicd"): logger = logging.getLogger(logger_name) for handler in logger.handlers[:]: if isinstance(handler, (logging.FileHandler, RotatingFileHandler)): handler.close() logger.removeHandler(handler) def _reset_logger(logger_name: str) -> None: """Reset a logger to clean state.""" logger = logging.getLogger(logger_name) logger.handlers = [] @pytest.fixture(autouse=True) def _clean_logging_state(): """Reset logging state before and after each test to release file locks on Windows.""" _close_all_file_handlers() for logger_name in ("", "fabric_cicd", "console_only"): _reset_logger(logger_name) yield _close_all_file_handlers() @pytest.fixture def temp_log_dir(): """Create a temporary directory for log files.""" tmpdir = Path(tempfile.mkdtemp()) yield tmpdir _close_all_file_handlers() shutil.rmtree(tmpdir, ignore_errors=True) @pytest.fixture def external_rotating_handler(temp_log_dir): """Create an external RotatingFileHandler for testing.""" log_file = temp_log_dir / "external.log" handler = RotatingFileHandler(str(log_file), maxBytes=1024, backupCount=1) handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")) yield handler handler.close() @pytest.fixture def external_logger_with_handler(temp_log_dir): """Create an external logger with a RotatingFileHandler attached.""" log_file = temp_log_dir / "external.log" handler = RotatingFileHandler(str(log_file), maxBytes=1024, backupCount=1) handler.setFormatter(logging.Formatter("%(message)s")) logger = logging.getLogger(f"ExternalLogger_{id(handler)}") logger.addHandler(handler) logger.setLevel(logging.DEBUG) yield logger, handler, log_file handler.close() logger.removeHandler(handler) class TestCustomFormatter: """Tests for the CustomFormatter class.""" @pytest.mark.parametrize( ("level", "level_name", "message"), [ (logging.DEBUG, "debug", "Debug message"), (logging.INFO, "info", "Info message"), (logging.WARNING, "warn", "Warning message"), (logging.ERROR, "error", "Error message"), (logging.CRITICAL, "crit", "Critical message"), ], ) def test_format_levels(self, level, level_name, message): """Test formatting of various log levels.""" formatter = CustomFormatter("[%(levelname)s] %(asctime)s - %(message)s", datefmt="%H:%M:%S") record = logging.LogRecord( name="fabric_cicd", level=level, pathname="", lineno=0, msg=message, args=(), exc_info=None, ) formatted = formatter.format(record) assert level_name in formatted.lower() assert message in formatted def test_format_with_indent(self): """Test formatting of messages with indent marker.""" formatter = CustomFormatter("[%(levelname)s] %(asctime)s - %(message)s", datefmt="%H:%M:%S") record = logging.LogRecord( name="fabric_cicd", level=logging.INFO, pathname="", lineno=0, msg=f"{constants.INDENT}Indented message", args=(), exc_info=None, ) formatted = formatter.format(record) assert "Indented message" in formatted assert formatted.startswith(" " * 8) class TestPackageFilter: """Tests for the PackageFilter class.""" @pytest.mark.parametrize( ("logger_name", "expected"), [ ("fabric_cicd", True), ("fabric_cicd.publish", True), ("fabric_cicd._common._logging", True), ("azure.identity", False), ("urllib3.connectionpool", False), ("other_package", False), ], ) def test_namespace_filtering(self, logger_name, expected): """Test filter correctly handles fabric_cicd and third-party namespaces.""" filter_instance = PackageFilter() record = logging.LogRecord( name=logger_name, level=logging.INFO, pathname="", lineno=0, msg="test", args=(), exc_info=None ) assert filter_instance.filter(record) is expected @pytest.mark.parametrize( ("level", "expected"), [ (logging.DEBUG, True), (logging.INFO, False), (logging.WARNING, False), (logging.ERROR, False), (logging.CRITICAL, False), ], ) def test_debug_only_mode(self, level, expected): """Test debug_only=True only allows DEBUG level from fabric_cicd.""" filter_instance = PackageFilter(debug_only=True) record = logging.LogRecord( name="fabric_cicd", level=level, pathname="", lineno=0, msg="test", args=(), exc_info=None ) assert filter_instance.filter(record) is expected def test_debug_only_still_checks_namespace(self): """Test debug_only=True still blocks non-fabric_cicd DEBUG logs.""" filter_instance = PackageFilter(debug_only=True) record = logging.LogRecord( name="azure.identity", level=logging.DEBUG, pathname="", lineno=0, msg="debug", args=(), exc_info=None ) assert filter_instance.filter(record) is False def test_default_allows_all_levels_from_package(self): """Test default filter (debug_only=False) allows all levels from fabric_cicd.""" filter_instance = PackageFilter(debug_only=False) levels = [logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, logging.CRITICAL] for level in levels: record = logging.LogRecord( name="fabric_cicd", level=level, pathname="", lineno=0, msg="test", args=(), exc_info=None ) assert filter_instance.filter(record) is True class TestMarkHandler: """Tests for the _mark_handler and _mark_external_handler functions.""" def test_mark_handler(self): """Test that _mark_handler sets attribute and returns same handler.""" handler = logging.StreamHandler() marked = _mark_handler(handler) assert getattr(marked, "_fabric_cicd_managed", False) is True assert marked is handler def test_mark_external_handler(self): """Test that _mark_external_handler sets attribute and returns same handler.""" handler = logging.StreamHandler() marked = _mark_external_handler(handler) assert getattr(marked, "_fabric_cicd_external", False) is True assert getattr(marked, "_fabric_cicd_managed", False) is False assert marked is handler class TestCleanupManagedHandlers: """Tests for the _cleanup_managed_handlers function.""" def test_removes_managed_preserves_external(self): """Test that managed handlers are removed while external are preserved.""" logger = logging.getLogger("test_cleanup") external_handler = logging.StreamHandler() managed_handler = _mark_handler(logging.StreamHandler()) logger.addHandler(external_handler) logger.addHandler(managed_handler) _cleanup_managed_handlers(logger) assert external_handler in logger.handlers assert managed_handler not in logger.handlers logger.removeHandler(external_handler) def test_cleanup_multiple_loggers(self): """Test cleanup across multiple loggers.""" logger_a = logging.getLogger("test_cleanup_a") logger_b = logging.getLogger("test_cleanup_b") handler_a = _mark_handler(logging.StreamHandler()) handler_b = _mark_handler(logging.StreamHandler()) logger_a.addHandler(handler_a) logger_b.addHandler(handler_b) _cleanup_managed_handlers(logger_a, logger_b) assert handler_a not in logger_a.handlers assert handler_b not in logger_b.handlers logger_a.handlers = [] logger_b.handlers = [] def test_cleanup_external_handler_removes_filters(self, temp_log_dir): """Test cleanup removes PackageFilter from external handlers.""" log_file = temp_log_dir / "external.log" external_handler = RotatingFileHandler(str(log_file), maxBytes=1024, backupCount=1) try: _mark_external_handler(external_handler) external_handler.addFilter(PackageFilter(debug_only=True)) root_logger = logging.getLogger() root_logger.addHandler(external_handler) assert len(external_handler.filters) == 1 assert isinstance(external_handler.filters[0], PackageFilter) _cleanup_managed_handlers(root_logger) assert external_handler not in root_logger.handlers assert len(external_handler.filters) == 0 assert getattr(external_handler, "_fabric_cicd_external", False) is False finally: external_handler.close() class TestConfigureDefaultFileHandler: """Tests for the _configure_default_file_handler function.""" def test_default_file_handler_configuration(self): """Test default file handler has correct configuration.""" handler = _configure_default_file_handler() try: assert isinstance(handler, logging.FileHandler) assert not isinstance(handler, RotatingFileHandler) assert getattr(handler, "_fabric_cicd_managed", False) is True assert handler.baseFilename.endswith("fabric_cicd.error.log") assert handler.mode == "w" assert handler.stream is None # delay=True assert len(handler.filters) == 1 assert isinstance(handler.filters[0], PackageFilter) assert handler.filters[0].debug_only is False assert handler.formatter is not None finally: handler.close() class TestConfigureExternalFileHandler: """Tests for the _configure_external_file_handler function.""" def test_reuses_handler_directly(self, temp_log_dir): """Test external file handler is reused directly (preserving rotation).""" log_file = temp_log_dir / "external.log" external_handler = RotatingFileHandler(str(log_file), maxBytes=1024, backupCount=1) custom_formatter = logging.Formatter("CUSTOM - %(message)s") external_handler.setFormatter(custom_formatter) try: handler = _configure_external_file_handler(external_handler, logging.DEBUG, debug_only_file=True) assert handler is external_handler assert isinstance(handler, RotatingFileHandler) assert handler.baseFilename == str(log_file) assert getattr(handler, "_fabric_cicd_managed", False) is False assert getattr(handler, "_fabric_cicd_external", False) is True assert handler.formatter is custom_formatter assert len(handler.filters) == 1 assert isinstance(handler.filters[0], PackageFilter) assert handler.filters[0].debug_only is True finally: external_handler.close() def test_preserves_caller_formatter(self, temp_log_dir): """Test external file handler preserves caller's formatter.""" log_file = temp_log_dir / "external.log" custom_formatter = logging.Formatter("CUSTOM - %(levelname)s - %(message)s") external_handler = RotatingFileHandler(str(log_file), maxBytes=1024, backupCount=1) external_handler.setFormatter(custom_formatter) try: handler = _configure_external_file_handler(external_handler, logging.DEBUG, debug_only_file=True) # Verify formatter is preserved by checking it formats correctly record = logging.LogRecord( name="fabric_cicd", level=logging.DEBUG, pathname="", lineno=0, msg="test", args=(), exc_info=None ) formatted = handler.formatter.format(record) assert formatted.startswith("CUSTOM - DEBUG - test") finally: external_handler.close() def test_info_level_ignores_debug_only(self, temp_log_dir): """Test external file handler at INFO level ignores debug_only_file flag.""" log_file = temp_log_dir / "external.log" external_handler = RotatingFileHandler(str(log_file), maxBytes=1024, backupCount=1) try: handler = _configure_external_file_handler(external_handler, logging.INFO, debug_only_file=True) assert handler.filters[0].debug_only is False finally: external_handler.close() def test_works_with_regular_file_handler(self, temp_log_dir): """Test external file handler works with regular FileHandler.""" log_file = temp_log_dir / "external.log" external_handler = logging.FileHandler(str(log_file)) try: handler = _configure_external_file_handler(external_handler, logging.DEBUG, debug_only_file=False) assert handler is external_handler assert isinstance(handler, logging.FileHandler) assert not isinstance(handler, RotatingFileHandler) assert len(handler.filters) == 1 assert handler.filters[0].debug_only is False finally: external_handler.close() class TestConfigureConsoleHandler: """Tests for the _configure_console_handler function.""" def test_console_handler_configuration(self): """Test console handler has correct configuration.""" handler = _configure_console_handler(logging.WARNING) assert isinstance(handler, logging.StreamHandler) assert handler.level == logging.WARNING assert getattr(handler, "_fabric_cicd_managed", False) is True assert isinstance(handler.formatter, CustomFormatter) class TestGetFileHandler: """Tests for the get_file_handler function.""" def test_returns_none_when_no_file_handler(self): """Test returns None when no file handler exists.""" assert get_file_handler() is None def test_returns_managed_file_handler_from_root(self): """Test returns the managed file handler from root logger.""" root_logger = logging.getLogger() handler = _mark_handler(logging.FileHandler("test_get.log", delay=True)) root_logger.addHandler(handler) try: result = get_file_handler() assert result is handler finally: handler.close() root_logger.removeHandler(handler) def test_ignores_unmanaged_file_handler_on_root(self): """Test ignores file handlers not marked as managed on root logger.""" root_logger = logging.getLogger() handler = logging.FileHandler("test_unmanaged.log", delay=True) root_logger.addHandler(handler) try: assert get_file_handler() is None finally: handler.close() root_logger.removeHandler(handler) def test_ignores_external_file_handler_on_root(self): """Test ignores file handlers marked as external on root logger.""" root_logger = logging.getLogger() handler = _mark_external_handler(logging.FileHandler("test_external.log", delay=True)) root_logger.addHandler(handler) try: assert get_file_handler() is None finally: handler.close() root_logger.removeHandler(handler) def test_returns_any_file_handler_from_provided_logger(self): """Test returns any file handler from provided logger.""" external_logger = logging.getLogger("external_test") handler = logging.FileHandler("test_external.log", delay=True) external_logger.addHandler(handler) try: result = get_file_handler(external_logger) assert result is handler finally: handler.close() external_logger.removeHandler(handler) class TestBuildConsoleMessage: """Tests for the _build_console_message function.""" def test_no_file_handler(self): """Test message without file handler reference.""" exception = Exception("Something failed") result = _build_console_message(exception, None) assert result == "Something failed" def test_with_default_file_handler(self): """Test message includes file path for default FileHandler.""" handler = logging.FileHandler("fabric_cicd.error.log", delay=True) try: exception = Exception("Something failed") result = _build_console_message(exception, handler) assert "Something failed" in result assert "See" in result assert "fabric_cicd.error.log" in result finally: handler.close() def test_with_non_default_file_handler(self, temp_log_dir): """Test message excludes file path for non-default file handlers.""" log_file = temp_log_dir / "program.log" handler = logging.FileHandler(str(log_file), delay=True) try: exception = Exception("Something failed") result = _build_console_message(exception, handler) assert result == "Something failed" assert "See" not in result finally: handler.close() class TestBuildFileMessage: """Tests for the _build_file_message function.""" @pytest.mark.parametrize( ("additional_info", "expected_in_result"), [ (None, False), ("status: 403", True), ], ) def test_file_message(self, additional_info, expected_in_result): """Test file message with and without additional info.""" exception = Exception("Something failed") if additional_info is not None: exception.additional_info = additional_info result = _build_file_message(exception) assert "%s" in result if expected_in_result: assert "Additional Info" in result assert additional_info in result else: assert result == "%s" class TestConfigureLogger: """Tests for the configure_logger function.""" @pytest.mark.parametrize( ("level", "expected_package_level", "expected_root_level"), [ (logging.INFO, logging.INFO, logging.ERROR), (logging.DEBUG, logging.DEBUG, logging.INFO), ], ) def test_logger_levels(self, level, expected_package_level, expected_root_level): """Test logger level configuration.""" configure_logger(level=level, disable_log_file=True) assert logging.getLogger("fabric_cicd").level == expected_package_level assert logging.getLogger().level == expected_root_level def test_default_includes_file_handler(self): """Test default configuration includes file handler.""" configure_logger() root_logger = logging.getLogger() file_handlers = [h for h in root_logger.handlers if isinstance(h, logging.FileHandler)] assert len(file_handlers) == 1 def test_disable_file_logging(self): """Test file logging can be disabled.""" configure_logger(disable_log_file=True) root_logger = logging.getLogger() file_handlers = [h for h in root_logger.handlers if isinstance(h, logging.FileHandler)] assert len(file_handlers) == 0 def test_with_external_file_handler(self, external_rotating_handler, temp_log_dir): """Test configuration with external file handler.""" log_file = temp_log_dir / "external.log" configure_logger( level=logging.DEBUG, external_file_handler=external_rotating_handler, suppress_debug_console=True, debug_only_file=True, ) root_logger = logging.getLogger() external_handlers = [ h for h in root_logger.handlers if isinstance(h, logging.FileHandler) and getattr(h, "_fabric_cicd_external", False) ] assert len(external_handlers) == 1 assert external_handlers[0] is external_rotating_handler assert external_handlers[0].baseFilename == str(log_file) assert isinstance(external_handlers[0], RotatingFileHandler) def test_suppress_debug_console(self): """Test suppressing DEBUG output to console.""" configure_logger(level=logging.DEBUG, suppress_debug_console=True, disable_log_file=True) package_logger = logging.getLogger("fabric_cicd") console_handlers = [h for h in package_logger.handlers if isinstance(h, logging.StreamHandler)] assert len(console_handlers) == 1 assert console_handlers[0].level == logging.INFO def test_console_only_logger_configured(self): """Test console_only logger is properly configured.""" configure_logger(disable_log_file=True) console_only_logger = logging.getLogger("console_only") package_logger = logging.getLogger("fabric_cicd") assert console_only_logger.propagate is False assert len(console_only_logger.handlers) == 1 assert package_logger.handlers[0] is not console_only_logger.handlers[0] def test_preserves_unmanaged_handlers(self): """Test that unmanaged handlers survive reconfiguration.""" root_logger = logging.getLogger() external_handler = logging.StreamHandler() root_logger.addHandler(external_handler) configure_logger(disable_log_file=True) configure_logger(disable_log_file=True) assert external_handler in root_logger.handlers root_logger.removeHandler(external_handler) def test_package_logger_propagates(self): """Test that package logger propagates to root.""" configure_logger(disable_log_file=True) assert logging.getLogger("fabric_cicd").propagate is True class TestLogHeader: """Tests for the log_header function.""" def test_logs_expected_messages(self, caplog): """Test log_header logs the expected messages.""" logger = logging.getLogger("fabric_cicd.test") logger.setLevel(logging.INFO) with caplog.at_level(logging.INFO, logger="fabric_cicd.test"): log_header(logger, "Test Header") assert len(caplog.records) >= 3 assert any("Test Header" in record.message for record in caplog.records) class TestWrapperFunctions: """Tests for the wrapper functions in __init__.py.""" @pytest.fixture(autouse=True) def _clear_feature_flags(self): """Clear feature flags before each wrapper test.""" constants.FEATURE_FLAG.clear() def test_append_feature_flag(self): """Test append_feature_flag adds flags correctly.""" append_feature_flag("feature_1") append_feature_flag("feature_2") append_feature_flag("feature_1") # Duplicate assert "feature_1" in constants.FEATURE_FLAG assert "feature_2" in constants.FEATURE_FLAG assert len([f for f in constants.FEATURE_FLAG if f == "feature_1"]) == 1 @pytest.mark.parametrize("level_input", ["DEBUG", "debug"]) def test_change_log_level(self, level_input): """Test change_log_level sets level correctly.""" change_log_level(level_input) assert logging.getLogger("fabric_cicd").level == logging.DEBUG def test_change_log_level_unsupported(self, capsys): """Test change_log_level warns on unsupported level.""" configure_logger(disable_log_file=True) change_log_level("TRACE") captured = capsys.readouterr() assert "not supported" in captured.err def test_disable_file_logging(self): """Test disable_file_logging removes file handlers.""" configure_logger() disable_file_logging() root_logger = logging.getLogger() file_handlers = [h for h in root_logger.handlers if isinstance(h, logging.FileHandler)] assert len(file_handlers) == 0 class TestConfigureExternalFileLogging: """Tests for the configure_external_file_logging wrapper function.""" def test_configures_correctly(self, external_logger_with_handler): """Test that configure_external_file_logging configures correctly.""" external_logger, external_handler, log_file = external_logger_with_handler configure_external_file_logging(external_logger) package_logger = logging.getLogger("fabric_cicd") assert package_logger.level == logging.DEBUG console_handlers = [ h for h in package_logger.handlers if isinstance(h, logging.StreamHandler) and not isinstance(h, logging.FileHandler) ] assert len(console_handlers) == 1 assert console_handlers[0].level == logging.INFO root_logger = logging.getLogger() external_handlers = [ h for h in root_logger.handlers if isinstance(h, logging.FileHandler) and getattr(h, "_fabric_cicd_external", False) ] assert len(external_handlers) == 1 assert external_handlers[0] is external_handler assert external_handlers[0].baseFilename == str(log_file) def test_raises_without_handler(self): """Test that configure_external_file_logging raises ValueError if no file handler.""" external_logger = logging.getLogger("NoFileHandler") external_logger.handlers = [] with pytest.raises(ValueError, match="No FileHandler or RotatingFileHandler found"): configure_external_file_logging(external_logger) def test_writes_only_debug_logs(self, external_logger_with_handler): """Test that only DEBUG logs from fabric_cicd are written to external file.""" external_logger, _external_handler, log_file = external_logger_with_handler configure_external_file_logging(external_logger) fabric_logger = logging.getLogger("fabric_cicd") fabric_logger.debug("Debug message") fabric_logger.info("Info message") azure_logger = logging.getLogger("azure.identity") azure_logger.setLevel(logging.DEBUG) azure_logger.debug("Azure debug") for handler in logging.getLogger().handlers: if hasattr(handler, "flush"): handler.flush() content = log_file.read_text(encoding="utf-8") assert "Debug message" in content assert "Info message" not in content assert "Azure debug" not in content class TestExceptionHandler: """Tests for the exception_handler function.""" def test_handles_custom_exception(self): """Test exception handler handles custom exceptions.""" from fabric_cicd._common._exceptions import InputError test_logger = logging.getLogger("fabric_cicd.test") exception = InputError("Test error message", logger=test_logger) configure_logger(disable_log_file=True) try: exception_handler(InputError, exception, None) except Exception: pytest.fail("exception_handler raised an unexpected exception") def test_falls_back_for_standard_exception(self): """Test exception handler falls back to default for standard exceptions.""" with patch("sys.__excepthook__") as mock_excepthook: exception = ValueError("Standard error") exception_handler(ValueError, exception, None) mock_excepthook.assert_called_once() def test_writes_to_console_only_logger(self): """Test that exception handler writes to console_only logger.""" from fabric_cicd._common._exceptions import InputError configure_logger(disable_log_file=True) test_logger = logging.getLogger("fabric_cicd.test") exception = InputError("User-facing error", logger=test_logger) with patch.object(logging.getLogger("console_only"), "error") as mock_error: exception_handler(InputError, exception, None) mock_error.assert_called_once() message = mock_error.call_args[0][0] assert "User-facing error" in message assert "See" not in message def test_removes_console_handler_when_using_default_file(self): """Test that exception handler removes console handler when using default file handler.""" from fabric_cicd._common._exceptions import InputError configure_logger() test_logger = logging.getLogger("fabric_cicd.test") exception = InputError("Test error", logger=test_logger) package_logger = logging.getLogger("fabric_cicd") assert len(package_logger.handlers) >= 1 exception_handler(InputError, exception, None) managed_handlers = [h for h in package_logger.handlers if getattr(h, "_fabric_cicd_managed", False)] assert len(managed_handlers) == 0 class TestFileLoggingIntegration: """Integration tests for file logging functionality.""" def test_default_file_handler_writes_logs(self, temp_log_dir): """Test that default file handler actually writes logs.""" import os original_cwd = Path.cwd() try: os.chdir(temp_log_dir) configure_logger() logger = logging.getLogger("fabric_cicd") logger.error("Error message for test") for handler in logging.getLogger().handlers: if hasattr(handler, "flush"): handler.flush() log_file = temp_log_dir / "fabric_cicd.error.log" assert log_file.exists() content = log_file.read_text(encoding="utf-8") assert "Error message for test" in content finally: os.chdir(original_cwd) def test_file_not_created_until_log_written(self, temp_log_dir): """Test that log file is not created until first log is written (delay=True).""" import os original_cwd = Path.cwd() try: os.chdir(temp_log_dir) configure_logger() log_file = temp_log_dir / "fabric_cicd.error.log" assert not log_file.exists() finally: os.chdir(original_cwd) def test_external_handler_writes_fabric_cicd_logs(self, external_logger_with_handler): """Test that external handler writes fabric_cicd logs to the shared file.""" external_logger, external_handler, log_file = external_logger_with_handler external_logger.debug("Program message 1") configure_external_file_logging(external_logger) fabric_logger = logging.getLogger("fabric_cicd") fabric_logger.debug("Fabric CICD message") for handler in logging.getLogger().handlers: if hasattr(handler, "flush"): handler.flush() external_handler.flush() content = log_file.read_text(encoding="utf-8") assert "Program message 1" in content assert "Fabric CICD message" in content def test_console_only_logger_does_not_propagate_to_file(self, external_logger_with_handler): """Test that console_only logger does not write to file.""" external_logger, _external_handler, log_file = external_logger_with_handler configure_external_file_logging(external_logger) console_only_logger = logging.getLogger("console_only") console_only_logger.error("Console only error") for handler in logging.getLogger().handlers: if hasattr(handler, "flush"): handler.flush() if log_file.exists(): content = log_file.read_text(encoding="utf-8") assert "Console only error" not in content class TestExternalHandlerReconfiguration: """Tests for external handler reconfiguration scenarios.""" def test_debug_to_non_debug_cleans_up_filter(self, external_logger_with_handler): """Test switching from debug mode to non-debug mode cleans up filter.""" external_logger, external_handler, _log_file = external_logger_with_handler configure_external_file_logging(external_logger) assert len(external_handler.filters) == 1 assert isinstance(external_handler.filters[0], PackageFilter) disable_file_logging() assert len(external_handler.filters) == 0 assert getattr(external_handler, "_fabric_cicd_external", False) is False def test_non_debug_to_debug_adds_filter(self, external_logger_with_handler): """Test switching from non-debug mode to debug mode adds filter correctly.""" external_logger, external_handler, _log_file = external_logger_with_handler disable_file_logging() assert len(external_handler.filters) == 0 configure_external_file_logging(external_logger) assert len(external_handler.filters) == 1 assert isinstance(external_handler.filters[0], PackageFilter) assert external_handler.filters[0].debug_only is True def test_multiple_debug_runs_no_filter_accumulation(self, external_logger_with_handler): """Test multiple debug runs don't accumulate filters on the same handler.""" external_logger, external_handler, _log_file = external_logger_with_handler configure_external_file_logging(external_logger) configure_external_file_logging(external_logger) configure_external_file_logging(external_logger) assert len(external_handler.filters) == 1 assert isinstance(external_handler.filters[0], PackageFilter) def test_handler_not_closed_on_disable(self, external_logger_with_handler): """Test external handler is not closed when file logging is disabled.""" external_logger, external_handler, log_file = external_logger_with_handler configure_external_file_logging(external_logger) disable_file_logging() external_logger.debug("Message after disable") external_handler.flush() content = log_file.read_text(encoding="utf-8") assert "Message after disable" in content def test_rotating_handler_preserves_rotation_settings(self, external_logger_with_handler): """Test that RotatingFileHandler rotation settings are preserved.""" external_logger, external_handler, _log_file = external_logger_with_handler original_max_bytes = external_handler.maxBytes original_backup_count = external_handler.backupCount configure_external_file_logging(external_logger) root_logger = logging.getLogger() external_handlers = [ h for h in root_logger.handlers if isinstance(h, RotatingFileHandler) and getattr(h, "_fabric_cicd_external", False) ] assert len(external_handlers) == 1 handler = external_handlers[0] assert handler.maxBytes == original_max_bytes assert handler.backupCount == original_backup_count ================================================ FILE: tests/test_parameter.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. import json import re from pathlib import Path from unittest import mock import pytest import yaml import fabric_cicd.constants as constants from fabric_cicd._parameter._parameter import Parameter SAMPLE_PARAMETER_FILE = """ find_replace: # Required Fields - find_value: "db52be81-c2b2-4261-84fa-840c67f4bbd0" replace_value: PPE: "81bbb339-8d0b-46e8-bfa6-289a159c0733" PROD: "5d6a1b16-447f-464a-b959-45d0fed35ca0" # Optional Fields item_type: "Notebook" item_name: ["Hello World"] file_path: "/Hello World.Notebook/notebook-content.py" spark_pool: # Required Fields - instance_pool_id: "72c68dbc-0775-4d59-909d-a47896f4573b" replace_value: PPE: type: "Capacity" name: "CapacityPool_Large_PPE" PROD: type: "Capacity" name: "CapacityPool_Large_PROD" # Optional Fields item_name: """ SAMPLE_PARAMETER_FILE_MULTIPLE = """ find_replace: # Required Fields - find_value: "db52be81-c2b2-4261-84fa-840c67f4bbd0" replace_value: PPE: "81bbb339-8d0b-46e8-bfa6-289a159c0733" PROD: "5d6a1b16-447f-464a-b959-45d0fed35ca0" # Optional Fields item_type: "Notebook" item_name: ["Hello World"] file_path: "/Hello World.Notebook/notebook-content.py" # Required Fields - find_value: "db52be81-c2b2-4261-84fa-840c67f4bbd0" replace_value: PPE: "81bbb339-8d0b-46e8-bfa6-289a159c0733" PROD: "5d6a1b16-447f-464a-b959-45d0fed35ca0" # Optional Fields item_type: "Notebook" item_name: ["Hello World"] file_path: "/Hello World.Notebook/notebook-content.py" key_value_replace: - find_key: $.variables[?(@.name=="SQL_Server")].value replace_value: PPE: "contoso-ppe.database.windows.net" PROD: "contoso-prod.database.windows.net" UAT: "contoso-uat.database.windows.net" # Optional fields: item_type: "VariableLibrary" item_name: "Vars" - find_key: $.variables[?(@.name=="Environment")].value replace_value: PPE: "PPE" PROD: "PROD" UAT: "UAT" # Optional fields: item_type: "VariableLibrary" item_name: "Vars" - find_key: $.variableOverrides[?(@.name=="SQL_Server")].value replace_value: PROD: "contoso-production-override.database.windows.net" file_path: Vars.VariableLibrary/valueSets/PROD.json item_type: "VariableLibrary" item_name: "Vars" - find_key: $.variableOverrides[?(@.name=="Environment")].value replace_value: PROD: "PROD_ENV" file_path: Vars.VariableLibrary/valueSets/PROD.json item_type: "VariableLibrary" item_name: "Vars" """ SAMPLE_INVALID_PARAMETER_FILE = """ find_replace: # Required Fields - find_value: "db52be81-c2b2-4261-84fa-840c67f4bbd0" replace_value: PPE: "81bbb339-8d0b-46e8-bfa6-289a159c0733" PROD: "5d6a1b16-447f-464a-b959-45d0fed35ca0" # Optional Fields item_type: "Notebook" item_name: ["Hello World"] file_path: "/Hello World.Notebook/notebook-content.py" spark_pool: # CapacityPool_Large "72c68dbc-0775-4d59-909d-a47896f4573b": type: "Capacity" name: "CapacityPool_Large" # CapacityPool_Medium "e7b8f1c4-4a6e-4b8b-9b2e-8f1e5d6a9c3d": type: "Workspace" name: "WorkspacePool_Medium" """ SAMPLE_PARAMETER_NO_TARGET_ENV = """ find_replace: # Required Fields - find_value: "db52be81-c2b2-4261-84fa-840c67f4bbd0" replace_value: DEV: "81bbb339-8d0b-46e8-bfa6-289a159c0733" PROD: "5d6a1b16-447f-464a-b959-45d0fed35ca0" # Optional Fields item_type: "Notebook" item_name: ["Hello World"] file_path: "/Hello World.Notebook/notebook-content.py" """ SAMPLE_PARAMETER_MISSING_FIND_VAL = """ find_replace: # Required Fields - find_value: replace_value: PPE: "81bbb339-8d0b-46e8-bfa6-289a159c0733" PROD: "5d6a1b16-447f-464a-b959-45d0fed35ca0" # Optional Fields item_type: "Notebook" item_name: ["Hello World"] file_path: "/Hello World.Notebook/notebook-content.py" """ SAMPLE_PARAMETER_MISMATCH_FILTER = """ find_replace: # Required Fields - find_value: "db52be81-c2b2-4261-84fa-840c67f4bbd0" replace_value: PPE: "81bbb339-8d0b-46e8-bfa6-289a159c0733" PROD: "5d6a1b16-447f-464a-b959-45d0fed35ca0" # Optional Fields item_type: "Notebook" item_name: ["Hello World", 'Hello World Subfolder'] file_path: "/Hello World.Notebook/notebook-content.py" """ SAMPLE_PARAMETER_MISSING_REPLACE_VAL = """ spark_pool: # Required Fields - instance_pool_id: "72c68dbc-0775-4d59-909d-a47896f4573b" replace_value: # Optional Fields item_name: """ SAMPLE_PARAMETER_INVALID_NAME = """ spark_pool_param: # Required Fields - instance_pool_id: "72c68dbc-0775-4d59-909d-a47896f4573b" replace_value: PPE: type: "Capacity" name: "CapacityPool_Large_PPE" PROD: type: "Capacity" name: "CapacityPool_Large_PROD" # Optional Fields item_name: """ SAMPLE_PARAMETER_INVALID_YAML_STRUC = """ spark_pool: # Required Fields instance_pool_id: "72c68dbc-0775-4d59-909d-a47896f4573b" replace_value: PPE: type: "Capacity" name: "CapacityPool_Large_PPE" PROD: type: "Capacity" name: "CapacityPool_Large_PROD" # Optional Fields item_name: """ SAMPLE_PARAMETER_INVALID_YAML_CHAR = """ find_replace: # Required Fields - find_value: '"db52be81-c2b2-4261-84fa-840c67f4bbd0" replace_value: PPE: "81bbb339-8d0b-46e8-bfa6-289a159c0733" PROD: "5d6a1b16-447f-464a-b959-45d0fed35ca0" # Optional Fields item_type: "Notebook" item_name: ["Hello World"] file_path: "/Hello World.Notebook/notebook-content.py" """ SAMPLE_PARAMETER_INVALID_IS_REGEX = """ find_replace: # Required Fields - find_value: "\\#\\s*META\\s+\"default_lakehouse\":\\s*\"([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})\"" replace_value: PPE: "81bbb339-8d0b-46e8-bfa6-289a159c0733" PROD: "5d6a1b16-447f-464a-b959-45d0fed35ca0" # Optional Fields is_regex: True item_type: "Notebook" """ SAMPLE_PARAMETER_ALL_ENV = """ find_replace: # Required Fields - find_value: "db52be81-c2b2-4261-84fa-840c67f4bbd0" replace_value: ALL: "universal-workspace-id-12345" # Optional Fields item_type: "Notebook" item_name: ["Hello World"] file_path: "/Hello World.Notebook/notebook-content.py" key_value_replace: - find_key: $.variables[?(@.name=="Environment")].value replace_value: ALL: "ANY_ENV" # Optional fields: item_type: "VariableLibrary" item_name: "Vars" """ SAMPLE_PLATFORM_FILE = """ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": { "type": "Notebook", "displayName": "Hello World", "description": "Sample notebook" }, "config": { "version": "2.0", "logicalId": "99b570c5-0c79-9dc4-4c9b-fa16c621384c" } } """ SAMPLE_NOTEBOOK_FILE = "print('Hello World and replace connection string: db52be81-c2b2-4261-84fa-840c67f4bbd0')" SAMPLE_PARAMETER_FILE_DUPLICATE_KEYS = """ find_replace: - find_value: "first-value" replace_value: PPE: "first-ppe" PROD: "first-prod" find_replace: - find_value: "second-value" replace_value: PPE: "second-ppe" PROD: "second-prod" """ SAMPLE_PARAMETER_FILE_MULTIPLE_DUPLICATE_KEYS = """ find_replace: - find_value: "first-value" replace_value: PPE: "first-ppe" spark_pool: - instance_pool_id: "pool-1" replace_value: PPE: type: "Capacity" name: "Pool1" find_replace: - find_value: "second-value" replace_value: PPE: "second-ppe" spark_pool: - instance_pool_id: "pool-2" replace_value: PPE: type: "Capacity" name: "Pool2" """ SAMPLE_PARAMETER_FILE_TRIPLE_DUPLICATE_KEY = """ find_replace: - find_value: "first-value" replace_value: PPE: "first-ppe" find_replace: - find_value: "second-value" replace_value: PPE: "second-ppe" find_replace: - find_value: "third-value" replace_value: PPE: "third-ppe" """ @pytest.fixture def item_type_in_scope(): return ["Notebook", "DataPipeline", "Environment"] @pytest.fixture def target_environment(): return "PPE" @pytest.fixture def repository_directory(tmp_path): # Create the sample workspace structure workspace_dir = tmp_path / "sample" / "workspace" workspace_dir.mkdir(parents=True, exist_ok=True) # Create the sample parameter file parameter_file_path = workspace_dir / constants.PARAMETER_FILE_NAME parameter_file_path.write_text(SAMPLE_PARAMETER_FILE) # Create sample invalid parameter files invalid_parameter_file_path = workspace_dir / "invalid_parameter.yml" invalid_parameter_file_path.write_text(SAMPLE_INVALID_PARAMETER_FILE) invalid_parameter_file_path1 = workspace_dir / "no_target_env_parameter.yml" invalid_parameter_file_path1.write_text(SAMPLE_PARAMETER_NO_TARGET_ENV) invalid_parameter_file_path2 = workspace_dir / "missing_find_val_parameter.yml" invalid_parameter_file_path2.write_text(SAMPLE_PARAMETER_MISSING_FIND_VAL) invalid_parameter_file_path3 = workspace_dir / "mismatch_filter_parameter.yml" invalid_parameter_file_path3.write_text(SAMPLE_PARAMETER_MISMATCH_FILTER) invalid_parameter_file_path4 = workspace_dir / "missing_replace_val_parameter.yml" invalid_parameter_file_path4.write_text(SAMPLE_PARAMETER_MISSING_REPLACE_VAL) invalid_parameter_file_path5 = workspace_dir / "invalid_name_parameter.yml" invalid_parameter_file_path5.write_text(SAMPLE_PARAMETER_INVALID_NAME) invalid_parameter_file_path6 = workspace_dir / "invalid_yaml_struc_parameter.yml" invalid_parameter_file_path6.write_text(SAMPLE_PARAMETER_INVALID_YAML_STRUC) invalid_parameter_file_path7 = workspace_dir / "invalid_yaml_char_parameter.yml" invalid_parameter_file_path7.write_text(SAMPLE_PARAMETER_INVALID_YAML_CHAR) invalid_parameter_file_path8 = workspace_dir / "invalid_is_regex_parameter.yml" invalid_parameter_file_path8.write_text(SAMPLE_PARAMETER_INVALID_IS_REGEX) # Create duplicate keys parameter files duplicate_keys_file = workspace_dir / "duplicate_keys_parameter.yml" duplicate_keys_file.write_text(SAMPLE_PARAMETER_FILE_DUPLICATE_KEYS) multiple_duplicate_keys_file = workspace_dir / "multiple_duplicate_keys_parameter.yml" multiple_duplicate_keys_file.write_text(SAMPLE_PARAMETER_FILE_MULTIPLE_DUPLICATE_KEYS) triple_duplicate_key_file = workspace_dir / "triple_duplicate_key_parameter.yml" triple_duplicate_key_file.write_text(SAMPLE_PARAMETER_FILE_TRIPLE_DUPLICATE_KEY) # Create the sample parameter file with ALL environment key all_env_parameter_file_path = workspace_dir / "all_env_parameter.yml" all_env_parameter_file_path.write_text(SAMPLE_PARAMETER_ALL_ENV) # Create the sample parameter file with multiple of a parameter multiple_parameter_file_path = workspace_dir / "multiple_parameter.yml" multiple_parameter_file_path.write_text(SAMPLE_PARAMETER_FILE_MULTIPLE) # Create the sample notebook files notebook_dir = workspace_dir / "Hello World.Notebook" notebook_dir.mkdir(parents=True, exist_ok=True) notebook_platform_file_path = notebook_dir / ".platform" notebook_platform_file_path.write_text(SAMPLE_PLATFORM_FILE) notebook_file_path = notebook_dir / "notebook-content.py" notebook_file_path.write_text(SAMPLE_NOTEBOOK_FILE) return workspace_dir @pytest.fixture def parameter_object(repository_directory, item_type_in_scope, target_environment): """Fixture to create a Parameter object.""" return Parameter( repository_directory=repository_directory, item_type_in_scope=item_type_in_scope, environment=target_environment, parameter_file_name=constants.PARAMETER_FILE_NAME, ) def test_parameter_class_initialization(parameter_object, repository_directory, item_type_in_scope, target_environment): """Test the Parameter class initialization.""" parameter_file_name = constants.PARAMETER_FILE_NAME # Check if the object is initialized correctly assert parameter_object.repository_directory == repository_directory assert parameter_object.item_type_in_scope == item_type_in_scope assert parameter_object.environment == target_environment assert parameter_object.parameter_file_name == parameter_file_name assert parameter_object.parameter_file_path == repository_directory / parameter_file_name def test_parameter_file_validation(parameter_object): """Test the validation methods for the parameter file""" assert parameter_object._validate_parameter_file_exists() == True assert parameter_object._validate_load_parameters_to_dict() == (True, parameter_object.environment_parameter) assert parameter_object._validate_parameter_load() == (True, constants.PARAMETER_MSGS["valid load"]) assert parameter_object._validate_parameter_names() == (True, constants.PARAMETER_MSGS["valid name"]) assert parameter_object._validate_parameter_structure() == (True, constants.PARAMETER_MSGS["valid structure"]) assert parameter_object._validate_parameter("find_replace") == ( True, constants.PARAMETER_MSGS["valid parameter"].format("find_replace"), ) assert parameter_object._validate_parameter("spark_pool") == ( True, constants.PARAMETER_MSGS["valid parameter"].format("spark_pool"), ) assert parameter_object._validate_parameter_file() == True def test_multiple_parameter_validation(repository_directory, item_type_in_scope, target_environment): """Test the validation methods for multiple parameters case""" multi_param_obj = Parameter( repository_directory=repository_directory, item_type_in_scope=item_type_in_scope, environment=target_environment, parameter_file_name="multiple_parameter.yml", ) assert multi_param_obj._validate_parameter("find_replace") == ( True, constants.PARAMETER_MSGS["valid parameter"].format("find_replace"), ) assert multi_param_obj._validate_parameter("key_value_replace") == ( True, constants.PARAMETER_MSGS["valid parameter"].format("key_value_replace"), ) assert multi_param_obj._validate_parameter_file() == True @pytest.mark.parametrize( ("param_name", "param_value", "result", "msg"), [ ("find_replace", ["find_value", "replace_value"], True, "valid keys"), ("find_replace", ["find_value", "item_type", "item_name", "file_path"], False, "missing key"), ("find_replace", ["find_value", "replace_value", "is_regex", "item_type"], True, "valid keys"), ("spark_pool", ["instance_pool_id", "replace_value", "item_name"], True, "valid keys"), ("spark_pool", ["instance_pool_id", "replace_value", "item_name", "file_path"], False, "invalid key"), ], ) def test_validate_parameter_keys(parameter_object, param_name, param_value, result, msg): """Test the validation methods for the find_replace parameter""" assert parameter_object._validate_parameter_keys(param_name, param_value) == ( result, constants.PARAMETER_MSGS[msg].format(param_name), ) @pytest.mark.parametrize(("param_name"), [("find_replace"), ("spark_pool")]) def test_validate_parameter(parameter_object, param_name): """Test the validation methods for a specific parameter""" param_dict = parameter_object.environment_parameter.get(param_name) for param in param_dict: assert parameter_object._validate_required_values(param_name, param) == ( True, constants.PARAMETER_MSGS["valid required values"].format(param_name), ) assert parameter_object._validate_replace_value(param_name, param["replace_value"]) == ( True, constants.PARAMETER_MSGS["valid replace value"].format(param_name), ) @pytest.mark.parametrize( ("replace_value", "result", "msg"), [ ( {"PPE": "81bbb339-8d0b-46e8-bfa6-289a159c0733", "PROD": "5d6a1b16-447f-464a-b959-45d0fed35ca0"}, True, "valid replace value", ), ( {"PPE": "81bbb339-8d0b-46e8-bfa6-289a159c0733", "PROD": None}, False, "missing replace value", ), ], ) def test_validate_find_replace_replace_value(parameter_object, replace_value, result, msg): """Test the _validate_find_replace_replace_value method.""" assert parameter_object._validate_find_replace_replace_value(replace_value) == ( result, constants.PARAMETER_MSGS[msg].format("find_replace", "PROD") if msg == "missing replace value" else constants.PARAMETER_MSGS[msg].format("find_replace"), ) @pytest.mark.parametrize( ("replace_value", "result", "msg"), [ # Valid cases - all values are same type ( {"PPE": "string_value", "PROD": "another_string"}, True, "valid replace value", ), ( {"PPE": True, "PROD": False}, True, "valid replace value", ), ( {"PPE": 123, "PROD": 456}, True, "valid replace value", ), ( {"PPE": 1.5, "PROD": 2.7}, True, "valid replace value", ), ( {"PPE": ["item1", "item2"], "PROD": ["item3", "item4"]}, True, "valid replace value", ), ( {"PPE": {"key": "value1"}, "PROD": {"key": "value2"}}, True, "valid replace value", ), # Invalid cases - missing values ( {"PPE": "value", "PROD": None}, False, "missing replace value", ), # Invalid cases - mixed types ( {"PPE": "string_value", "PROD": 123}, False, "mixed types", ), ( {"PPE": True, "PROD": "false"}, False, "mixed types", ), ( {"PPE": 123, "PROD": 45.6}, False, "mixed types", ), ], ) def test_validate_key_value_replace_replace_value(parameter_object, replace_value, result, msg): """Test the _validate_key_value_replace_replace_value method.""" is_valid, actual_msg = parameter_object._validate_key_value_replace_replace_value(replace_value) if msg == "valid replace value": expected_msg = constants.PARAMETER_MSGS[msg].format("key_value_replace") assert (is_valid, actual_msg) == (result, expected_msg) elif msg == "missing replace value": # For missing replace value, check that the message contains the expected format assert is_valid == result assert "key_value_replace is missing a replace value for" in actual_msg elif msg == "mixed types": # For mixed types, check that the message contains the expected content assert is_valid == result assert "Inconsistent data types in key_value_replace replace_value" in actual_msg @pytest.mark.parametrize( ("replace_value", "result", "msg", "desc"), [ ( { "PPE": {"type": "Capacity", "name": "CapacityPool_Large_PPE"}, "PROD": {"type": "Capacity", "name": "CapacityPool_Large_PROD"}, }, True, "valid replace value", None, ), ( { "PPE": {}, "PROD": {"type": "Capacity", "name": "CapacityPool_Large_PROD"}, }, False, "missing replace value", None, ), ( { "PPE": {"name": "CapacityPool_Large_PPE"}, "PROD": {"type": "Capacity", "name": "CapacityPool_Large_PROD"}, }, False, "invalid replace value", "missing key", ), ( { "PPE": {"type": "Capacity", "name": "CapacityPool_Large_PPE"}, "PROD": {"type": "Capacity", "name": None}, }, False, "invalid replace value", "missing value", ), ( { "PPE": {"type": "Capacity", "name": "CapacityPool_Large_PPE"}, "PROD": {"type": "Test", "name": "CapacityPool_Large_PROD"}, }, False, "invalid replace value", "invalid value", ), ], ) def test_validate_spark_pool_replace_value(parameter_object, replace_value, result, msg, desc): """Test the _validate_spark_pool_replace_value method.""" if msg == "valid replace value": msg = constants.PARAMETER_MSGS[msg].format("spark_pool") if msg == "missing replace value": msg = constants.PARAMETER_MSGS[msg].format("spark_pool", "PPE") if msg == "invalid replace value" and desc == "missing key": msg = constants.PARAMETER_MSGS[msg][desc].format("PPE") if msg == "invalid replace value" and desc == "missing value": msg = constants.PARAMETER_MSGS[msg][desc].format("PROD", "name") if msg == "invalid replace value" and desc == "invalid value": msg = constants.PARAMETER_MSGS[msg][desc].format("PROD") assert parameter_object._validate_spark_pool_replace_value(replace_value) == (result, msg) assert parameter_object._validate_replace_value( "spark_pool", { "PPE": {}, "PROD": {"type": "Capacity", "name": "CapacityPool_Large_PROD"}, }, ) == (False, constants.PARAMETER_MSGS["missing replace value"].format("spark_pool", "PPE")) def test_validate_data_type(parameter_object): """Test data type validation""" # General data type validation assert parameter_object._validate_data_type([1, 2, 3], "string or list[string]", "key", "param_name") == ( False, constants.PARAMETER_MSGS["invalid data type"].format("key", "string or list[string]", "param_name"), ) required_values = { "find_value": ["db52be81-c2b2-4261-84fa-840c67f4bbd0"], "replace_value": { "PPE": "81bbb339-8d0b-46e8-bfa6-289a159c0733", "PROD": "5d6a1b16-447f-464a-b959-45d0fed35ca0", }, } # Data type error in required values assert parameter_object._validate_required_values("find_replace", required_values) == ( False, constants.PARAMETER_MSGS["invalid data type"].format("find_value", "string", "find_replace"), ) find_replace_value = { "PPE": "81bbb339-8d0b-46e8-bfa6-289a159c0733", "PROD": 123, } # Data type error in find_replace replace value dict assert parameter_object._validate_find_replace_replace_value(find_replace_value) == ( False, constants.PARAMETER_MSGS["invalid data type"].format("PROD replace_value", "string", "find_replace"), ) spark_pool_replace_value_1 = { "PPE": "string", "PROD": {"type": "Capacity", "name": "CapacityPool_Large_PROD"}, } # Data type error in spark_pool replace value dict assert parameter_object._validate_spark_pool_replace_value(spark_pool_replace_value_1) == ( False, constants.PARAMETER_MSGS["invalid data type"].format("PPE key", "dictionary", "spark_pool"), ) spark_pool_replace_value_2 = { "PPE": {"type": "Capacity", "name": "CapacityPool_Large_PPE"}, "PROD": {"type": ["Capacity"], "name": "CapacityPool_Large_PROD"}, } # Data type error in spark_pool replace value environment dict assert parameter_object._validate_spark_pool_replace_value(spark_pool_replace_value_2) == ( False, constants.PARAMETER_MSGS["invalid data type"].format("type", "string", "spark_pool"), ) param_dict = { "item_type": "Notebook", "item_name": {"Hello World"}, "file_path": "/Hello World.Notebook/notebook-content.py", } # Data type error in optional values assert parameter_object._validate_optional_values("find_replace", param_dict) == ( False, constants.PARAMETER_MSGS["invalid data type"].format("item_name", "string or list[string]", "find_replace"), ) def test_validate_yaml_content_empty(): """Test that empty YAML content is handled correctly.""" import tempfile # Test empty content - create a temp file with empty content with tempfile.NamedTemporaryFile(mode="w", suffix=".yml", delete=False) as temp_file: temp_file.write("\n\n\n\t") temp_file_path = temp_file.name try: param = Parameter( repository_directory=Path(temp_file_path).parent, item_type_in_scope=["Notebook"], environment="TEST", parameter_file_name=Path(temp_file_path).name, ) # Empty content should fail to load assert not param.environment_parameter, "Empty YAML should not load successfully" is_valid, msg = param._validate_parameter_load() assert is_valid is False assert constants.PARAMETER_MSGS["empty yaml"] in msg finally: Path(temp_file_path).unlink() def test_utf8_validation_at_file_read(): """Test that Python validates UTF-8 encoding when reading parameter files.""" import tempfile # Create a file with invalid UTF-8 bytes with tempfile.NamedTemporaryFile(mode="wb", suffix=".yml", delete=False) as temp_file: # Write valid YAML structure with invalid UTF-8 byte sequence temp_file.write(b"find_replace:\n - find_value: 'invalid \x80\x81\x82 bytes'\n") temp_file_path = temp_file.name try: # Parameter creation should succeed but loading should fail gracefully param = Parameter( repository_directory=Path(temp_file_path).parent, item_type_in_scope=["Notebook"], environment="TEST", parameter_file_name=Path(temp_file_path).name, ) # Parameter should fail to load due to invalid UTF-8 assert not param.environment_parameter, "Invalid UTF-8 file should not load successfully" # Verify that validation reports the file as invalid is_valid, msg = param._validate_parameter_load() assert is_valid is False, "Parameter load should fail for invalid UTF-8" # Verify the error message uses the constant format with UnicodeDecodeError details assert constants.PARAMETER_MSGS["invalid load"].split("{}")[0] in msg, ( f"Error message should use 'invalid load' constant format: {msg}" ) # UnicodeDecodeError message contains 'utf-8' and 'decode' assert "utf-8" in msg.lower(), f"Error message should contain UTF-8 details: {msg}" finally: # Clean up temporary file Path(temp_file_path).unlink() def test_validate_yaml_content_duplicate_keys(): """Test that duplicate keys are detected via _DuplicateKeyLoader.""" import tempfile # Test duplicate key detection - create a temp file with duplicate keys with tempfile.NamedTemporaryFile(mode="w", suffix=".yml", delete=False) as temp_file: temp_file.write("""find_replace: - find_value: "first" replace_value: TEST: "first-value" find_replace: - find_value: "second" replace_value: TEST: "second-value" """) temp_file_path = temp_file.name try: param = Parameter( repository_directory=Path(temp_file_path).parent, item_type_in_scope=["Notebook"], environment="TEST", parameter_file_name=Path(temp_file_path).name, ) # Duplicate keys should fail to load assert not param.environment_parameter, "YAML with duplicate keys should not load successfully" is_valid, msg = param._validate_parameter_load() assert is_valid is False assert constants.PARAMETER_MSGS["duplicate key"].split("{}")[0].lower() in msg.lower() finally: Path(temp_file_path).unlink() def test_duplicate_keys_single_duplicate(repository_directory, item_type_in_scope, target_environment): """Test detection of a single duplicate root-level key via _DuplicateKeyLoader.""" param_obj = Parameter( repository_directory=repository_directory, item_type_in_scope=item_type_in_scope, environment=target_environment, parameter_file_name="duplicate_keys_parameter.yml", ) # Duplicate keys should fail to load - _DuplicateKeyLoader catches them during yaml.load() assert not param_obj.environment_parameter, "YAML with duplicate keys should not load successfully" is_valid, _msg = param_obj._validate_parameter_load() assert is_valid is False assert constants.PARAMETER_MSGS["duplicate key"].split("{}")[0].lower() in _msg.lower() def test_duplicate_keys_multiple_duplicates(repository_directory, item_type_in_scope, target_environment): """Test detection of multiple duplicate root-level keys via _DuplicateKeyLoader.""" param_obj = Parameter( repository_directory=repository_directory, item_type_in_scope=item_type_in_scope, environment=target_environment, parameter_file_name="multiple_duplicate_keys_parameter.yml", ) # Duplicate keys should fail to load - _DuplicateKeyLoader catches them during yaml.load() assert not param_obj.environment_parameter, "YAML with duplicate keys should not load successfully" is_valid, _msg = param_obj._validate_parameter_load() assert is_valid is False assert constants.PARAMETER_MSGS["duplicate key"].split("{}")[0].lower() in _msg.lower() def test_duplicate_keys_no_duplicates(parameter_object): """Test that valid YAML with no duplicate keys passes.""" # parameter_object fixture uses a valid parameter file without duplicates assert parameter_object.environment_parameter, "Valid YAML should load successfully" is_valid, _msg = parameter_object._validate_parameter_load() assert is_valid is True def test_duplicate_keys_ignores_comments(): """Test that comment lines starting with # are ignored.""" import tempfile content = """ # This is a comment find_replace: - find_value: "value1" replace_value: PPE: "ppe-value" # Another comment # find_replace: this should be ignored spark_pool: - instance_pool_id: "pool-id" replace_value: PPE: type: "Capacity" name: "Pool" """ with tempfile.NamedTemporaryFile(mode="w", suffix=".yml", delete=False) as temp_file: temp_file.write(content) temp_file_path = temp_file.name try: param = Parameter( repository_directory=Path(temp_file_path).parent, item_type_in_scope=["Notebook"], environment="PPE", parameter_file_name=Path(temp_file_path).name, ) # Valid YAML with comments should load successfully assert param.environment_parameter, "Valid YAML with comments should load successfully" is_valid, _msg = param._validate_parameter_load() assert is_valid is True finally: Path(temp_file_path).unlink() def test_duplicate_keys_nested_keys_not_flagged(): """Test that nested keys (indented) are not flagged as root-level duplicates.""" import tempfile content = """ find_replace: - find_value: "value1" replace_value: PPE: "ppe-value" - find_value: "value2" replace_value: PPE: "ppe-value2" """ with tempfile.NamedTemporaryFile(mode="w", suffix=".yml", delete=False) as temp_file: temp_file.write(content) temp_file_path = temp_file.name try: param = Parameter( repository_directory=Path(temp_file_path).parent, item_type_in_scope=["Notebook"], environment="PPE", parameter_file_name=Path(temp_file_path).name, ) # Valid YAML with nested repeated keys should load successfully assert param.environment_parameter, "Valid YAML with nested keys should load successfully" is_valid, _msg = param._validate_parameter_load() assert is_valid is True finally: Path(temp_file_path).unlink() @pytest.mark.parametrize( ("content", "duplicate_keys"), [ # Duplicate environment key within replace_value ( """ find_replace: - find_value: "value1" replace_value: PPE: "ppe-value1" PPE: "ppe-value2" """, ["PPE"], ), # Duplicate find_value within a single list entry ( """ find_replace: - find_value: "first" find_value: "second" replace_value: PPE: "ppe-value" """, ["find_value"], ), # Duplicate replace_value within a single list entry ( """ find_replace: - find_value: "test-id" replace_value: PPE: "first-ppe" replace_value: PPE: "second-ppe" """, ["replace_value"], ), # Duplicate optional field (item_type) ( """ find_replace: - find_value: "test-id" replace_value: PPE: "ppe-value" item_type: "Notebook" item_type: "DataPipeline" """, ["item_type"], ), # Case-insensitive duplicate environment key ( """ find_replace: - find_value: "abc" replace_value: PROD: "value1" prod: "value2" """, ["prod"], ), # Duplicate item_name within a single list entry ( """ find_replace: - find_value: "test-id" replace_value: PPE: "ppe-value" item_name: "Notebook1" item_name: "Notebook2" """, ["item_name"], ), # Duplicate file_path within a single list entry ( """ find_replace: - find_value: "test-id" replace_value: PPE: "ppe-value" file_path: "/path/one.py" file_path: "/path/two.py" """, ["file_path"], ), # Multiple different duplicate keys in the same entry ( """ find_replace: - find_value: "test-id" find_value: "test-id-2" replace_value: PPE: "first-ppe" replace_value: PPE: "second-ppe" item_type: "Notebook" item_type: "DataPipeline" """, ["find_value", "replace_value", "item_type"], ), ], ids=[ "duplicate_environment_key", "duplicate_find_value", "duplicate_replace_value", "duplicate_item_type", "case_insensitive_duplicate", "duplicate_item_name", "duplicate_file_path", "multiple_different_duplicate_keys", ], ) def test_duplicate_keys_nested_duplicate_detected(content, duplicate_keys): """Test that duplicate keys within nested mappings are detected by _DuplicateKeyLoader.""" import tempfile with tempfile.NamedTemporaryFile(mode="w", suffix=".yml", delete=False) as temp_file: temp_file.write(content) temp_file_path = temp_file.name try: param = Parameter( repository_directory=Path(temp_file_path).parent, item_type_in_scope=["Notebook"], environment="PPE", parameter_file_name=Path(temp_file_path).name, ) # Duplicate keys should fail to load assert not param.environment_parameter, ( f"YAML with duplicate '{duplicate_keys}' keys should not load successfully" ) is_valid, msg = param._validate_parameter_load() assert is_valid is False for key in duplicate_keys: assert key.lower() in msg.lower(), f"Expected '{key}' in error message: {msg}" finally: Path(temp_file_path).unlink() def test_duplicate_keys_triple_occurrence(repository_directory, item_type_in_scope, target_environment): """Test detection of a key appearing more than twice via _DuplicateKeyLoader.""" param_obj = Parameter( repository_directory=repository_directory, item_type_in_scope=item_type_in_scope, environment=target_environment, parameter_file_name="triple_duplicate_key_parameter.yml", ) # Duplicate keys should fail to load - _DuplicateKeyLoader catches them during yaml.load() assert not param_obj.environment_parameter, "YAML with duplicate keys should not load successfully" is_valid, msg = param_obj._validate_parameter_load() assert is_valid is False assert constants.PARAMETER_MSGS["duplicate key"].split("{}")[0].lower() in msg.lower() def test_validate_parameter_file_structure(repository_directory, item_type_in_scope, target_environment): """Test the validation of the parameter file structure""" param_obj = Parameter( repository_directory=repository_directory, item_type_in_scope=item_type_in_scope, environment=target_environment, parameter_file_name="invalid_parameter.yml", ) assert param_obj._validate_parameter_structure() == (False, constants.PARAMETER_MSGS["invalid structure"]) def test_validate_optional_values(parameter_object): """Test the _validate_optional_values method.""" param_dict_1 = { "item_type": "Notebook", "item_name": ["Hello World"], "file_path": "/Hello World.Notebook/notebook-content.py", } assert parameter_object._validate_optional_values("find_replace", param_dict_1) == ( True, constants.PARAMETER_MSGS["valid optional"].format("find_replace"), ) param_dict_2 = { "item_type": "SparkNotebook", "item_name": ["Hello World"], "file_path": "/Hello World.Notebook/notebook-content.py", } assert parameter_object._validate_optional_values("find_replace", param_dict_2, check_match=True) == ( False, "no match", ) param_dict_3 = {"item_name": "Hello World"} assert parameter_object._validate_optional_values("spark_pool", param_dict_3, check_match=True) == ( True, constants.PARAMETER_MSGS["valid optional"].format("spark_pool"), ) @pytest.mark.parametrize( ("param_name"), ["find_replace", "spark_pool"], ) def test_validate_parameter_environment_and_filters(parameter_object, param_name): """Test the validation methods for environment and filters""" for param_dict in parameter_object.environment_parameter.get(param_name): # Environment validation assert parameter_object._validate_environment(param_dict["replace_value"]) == (True, "env") # Optional filters validation assert parameter_object._validate_item_type("Pipeline") == ( False, constants.PARAMETER_MSGS["invalid item type"].format("Pipeline"), ) assert parameter_object._validate_item_name("Hello World 2") == ( False, constants.PARAMETER_MSGS["invalid item name"].format("Hello World 2"), ) assert parameter_object._validate_file_path(["Hello World 2.Notebook/notebook-content.py"]) == ( False, constants.PARAMETER_MSGS["no valid file path"].format(["Hello World 2.Notebook/notebook-content.py"]), ) def test_validate_item_name_with_accented_characters(repository_directory, item_type_in_scope, target_environment): """Test that _validate_item_name correctly handles item names with accented characters.""" # Create a notebook directory with accented characters in the displayName accented_name = "Rôles et parcours de formation" accented_dir = repository_directory / f"{accented_name}.Report" accented_dir.mkdir(parents=True, exist_ok=True) platform_content = f"""{{ "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", "metadata": {{ "type": "Report", "displayName": "{accented_name}", "description": "Report with accented characters" }}, "config": {{ "version": "2.0", "logicalId": "99b570c5-0c79-9dc4-4c9b-fa16c621384c" }} }} """ platform_file = accented_dir / ".platform" platform_file.write_text(platform_content, encoding="utf-8") param_obj = Parameter( repository_directory=repository_directory, item_type_in_scope=item_type_in_scope, environment=target_environment, parameter_file_name=constants.PARAMETER_FILE_NAME, ) # Should find the item with accented characters assert param_obj._validate_item_name(accented_name) == (True, "Valid item name") # Should not find a non-existent item assert param_obj._validate_item_name("Roles et parcours de formation") == ( False, constants.PARAMETER_MSGS["invalid item name"].format("Roles et parcours de formation"), ) @pytest.mark.parametrize( ("param_file_name", "result", "msg"), [ ("no_target_env_parameter.yml", True, "valid parameter"), ("missing_find_val_parameter.yml", False, "missing required value"), ("mismatch_filter_parameter.yml", True, "valid parameter"), ("missing_replace_val_parameter.yml", False, "missing required value"), ("invalid_name_parameter.yml", False, "invalid name"), ("invalid_yaml_struc_parameter.yml", False, "invalid load"), ("invalid_yaml_char_parameter.yml", False, "invalid load"), ("invalid_is_regex_parameter.yml", False, "invalid data type"), ], ) def test_validate_invalid_parameters( repository_directory, item_type_in_scope, target_environment, param_file_name, result, msg ): """Test the validation of invalid or error-prone parameter files""" param_obj = Parameter( repository_directory=repository_directory, item_type_in_scope=item_type_in_scope, environment=target_environment, parameter_file_name=param_file_name, ) # Target environment not present in find_replace parameter (error-prone case) if param_file_name == "no_target_env_parameter.yml": assert param_obj._validate_parameter("find_replace") == ( result, constants.PARAMETER_MSGS[msg].format("find_replace"), ) # Missing required value in find_replace parameter if param_file_name == "missing_find_val_parameter.yml": for param_dict in param_obj.environment_parameter.get("find_replace"): assert param_obj._validate_required_values("find_replace", param_dict) == ( result, constants.PARAMETER_MSGS[msg].format("find_value", "find_replace"), ) assert param_obj._validate_parameter_file() == result # Mismatched optional filters in find_replace parameter (error-prone case) if param_file_name == "mismatch_filter_parameter.yml": assert param_obj._validate_parameter("find_replace") == ( result, constants.PARAMETER_MSGS[msg].format("find_replace"), ) # Missing required value in spark_pool parameter if param_file_name == "missing_replace_val_parameter.yml": assert param_obj._validate_parameter("spark_pool") == ( result, constants.PARAMETER_MSGS[msg].format("replace_value", "spark_pool"), ) # Invalid parameter name if param_file_name == "invalid_name_parameter.yml": assert param_obj._validate_parameter_names() == ( result, constants.PARAMETER_MSGS[msg].format("spark_pool_param"), ) # Errors in YAML content structure if param_file_name == "invalid_yaml_struc_parameter.yml": is_valid, msg = param_obj._validate_parameter_load() try: with Path.open(repository_directory / param_file_name, encoding="utf-8") as yaml_file: yaml_content = yaml_file.read() yaml.full_load(yaml_content) except yaml.YAMLError as e: error_message = str(e) assert is_valid == result assert msg == constants.PARAMETER_MSGS["invalid load"].format(error_message) # Mismatched quotes in YAML content if param_file_name == "invalid_yaml_char_parameter.yml": is_valid, msg = param_obj._validate_parameter_load() try: with Path.open(repository_directory / param_file_name, encoding="utf-8") as yaml_file: yaml_content = yaml_file.read() yaml.full_load(yaml_content) except yaml.YAMLError as e: error_message = str(e) assert is_valid == result assert msg == constants.PARAMETER_MSGS["invalid load"].format(error_message) # Invalid is_regex value in find_replace parameter if param_file_name == "invalid_is_regex_parameter.yml": # Mock the environment_parameter to have the invalid is_regex (boolean instead of string) param_obj.environment_parameter = { "find_replace": [ { "find_value": '\\#\\s*META\\s+"default_lakehouse":\\s*"([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})"', "replace_value": { "PPE": "81bbb339-8d0b-46e8-bfa6-289a159c0733", "PROD": "5d6a1b16-447f-464a-b959-45d0fed35ca0", }, "is_regex": True, # This is a boolean, not a string } ] } assert param_obj._validate_parameter("find_replace") == ( result, constants.PARAMETER_MSGS[msg].format("is_regex", "string", "find_replace"), ) def test_validate_file_path_scenarios(parameter_object): """Test _validate_file_path with different scenarios.""" # Test 1: Single invalid path - expects "no valid file path" error single_invalid_path = ["nonexistent_file.py"] with mock.patch("fabric_cicd._parameter._utils.process_input_path", return_value=[]): result, msg = parameter_object._validate_file_path(single_invalid_path) assert result is False assert msg == constants.PARAMETER_MSGS["no valid file path"].format(single_invalid_path) # Test 2: Multiple invalid paths - expects "no valid file path" error multiple_invalid_paths = ["nonexistent_file1.py", "nonexistent_file2.py"] with mock.patch("fabric_cicd._parameter._utils.process_input_path", return_value=[]): result, msg = parameter_object._validate_file_path(multiple_invalid_paths) assert result is False assert msg == constants.PARAMETER_MSGS["no valid file path"].format(multiple_invalid_paths) # Test 3: Mixed valid/invalid paths - expects "invalid file path" error for missing paths mixed_paths = ["valid_path.py", "invalid_path.py"] # Create a custom test implementation that simulates the behavior we want to test def mock_validate_file_path(input_path): """Custom implementation to test the mixed valid/invalid paths case""" # For test purposes, we'll simulate that process_input_path returned a valid path valid_paths = [Path("valid_path.py")] # If there are no valid paths, return the "no valid file path" error if not valid_paths: return False, constants.PARAMETER_MSGS["no valid file path"].format(input_path) # Normalize paths for comparison processed_paths = {str(p).replace("\\", "/") for p in valid_paths} original_paths = {str(p).replace("\\", "/") for p in input_path} # Find invalid paths missing_paths = original_paths - processed_paths if missing_paths: path_diff = len(original_paths) - len(processed_paths) return False, constants.PARAMETER_MSGS["invalid file path"].format(input_path, path_diff) return True, "Valid file path" # Save the original method original_method = parameter_object._validate_file_path # Replace with our mock implementation parameter_object._validate_file_path = mock_validate_file_path try: # Call the method with our test data result, msg = parameter_object._validate_file_path(mixed_paths) # Should return False and the invalid file path message assert result is False # The message should contain the invalid path assert "invalid_path.py" in msg # Make sure we're getting the specific "invalid file path" message # We need to match the new format which takes input_path and path_diff mixed_paths = ["valid_path.py", "invalid_path.py"] path_diff = 1 # One invalid path expected_msg = constants.PARAMETER_MSGS["invalid file path"].format(mixed_paths, path_diff) assert msg == expected_msg finally: # Restore the original method parameter_object._validate_file_path = original_method # Test 4: All valid paths - should return True valid_paths = ["valid_path1.py", "valid_path2.py"] # Create a mock function that always returns success def mock_validate_all_valid(_): """Mock function that simulates all paths being valid""" return True, "Valid file path" # Save the original method and replace with our mock original_method = parameter_object._validate_file_path parameter_object._validate_file_path = mock_validate_all_valid try: # Call the method with our test data result, msg = parameter_object._validate_file_path(valid_paths) assert result is True assert msg == "Valid file path" finally: # Restore the original method parameter_object._validate_file_path = original_method def test_validate_all_environment_key_valid(): """Test the validation of _ALL_ environment key in valid scenarios""" # Test that _ALL_ key is accepted as a valid environment import tempfile from fabric_cicd._parameter._parameter import Parameter # Create a temporary parameter file with _ALL_ environment key with tempfile.NamedTemporaryFile(mode="w", suffix=".yml", delete=False) as temp_file: temp_file.write(""" find_replace: - find_value: "test-value" replace_value: _ALL_: "universal-value" key_value_replace: - find_key: $.test replace_value: _All_: "universal-key-value" spark_pool: - instance_pool_id: "test-pool-id" replace_value: _all_: type: "Capacity" name: "UniversalPool" """) temp_file_path = temp_file.name try: param_obj = Parameter( repository_directory=Path(temp_file_path).parent, item_type_in_scope=["Notebook"], environment="TEST", parameter_file_name=Path(temp_file_path).name, ) # Should pass validation since _ALL_ is a valid environment key assert param_obj._validate_parameter("find_replace") == ( True, constants.PARAMETER_MSGS["valid parameter"].format("find_replace"), ) assert param_obj._validate_parameter("key_value_replace") == ( True, constants.PARAMETER_MSGS["valid parameter"].format("key_value_replace"), ) assert param_obj._validate_parameter("spark_pool") == ( True, constants.PARAMETER_MSGS["valid parameter"].format("spark_pool"), ) # Overall parameter file should be valid assert param_obj._validate_parameter_file() == True # Test that the _ALL_ environment key is properly recognized (case-insensitive) for param_dict in param_obj.environment_parameter.get("find_replace"): assert param_obj._validate_environment(param_dict["replace_value"]) == (True, "_ALL_") for param_dict in param_obj.environment_parameter.get("key_value_replace"): assert param_obj._validate_environment(param_dict["replace_value"]) == (True, "_All_") for param_dict in param_obj.environment_parameter.get("spark_pool"): assert param_obj._validate_environment(param_dict["replace_value"]) == (True, "_all_") finally: # Clean up temporary file Path(temp_file_path).unlink() def test_validate_all_environment_key_invalid(): """Test validation of _ALL_ environment key in invalid scenarios""" import tempfile from fabric_cicd._parameter._parameter import Parameter # Create a parameter file with multiple environment keys including ALL with tempfile.NamedTemporaryFile(mode="w", suffix=".yml", delete=False) as temp_file: temp_file.write(""" find_replace: - find_value: "test-connection-string" replace_value: DEV: "dev-connection-string" TEST: "test-connection-string" PROD: "prod-connection-string" _ALL_: "universal-connection-string" spark_pool: - instance_pool_id: "multi-env-pool-id" replace_value: DEV: type: "Workspace" name: "DevPool" TEST: type: "Workspace" name: "TestPool" PROD: type: "Capacity" name: "ProdPool" _all_: type: "Capacity" name: "UniversalPool" """) temp_file_path = temp_file.name try: param_obj = Parameter( repository_directory=Path(temp_file_path).parent, item_type_in_scope=["Notebook"], environment="TEST", parameter_file_name=Path(temp_file_path).name, ) # Should fail validation since ALL cannot coexist with other environment keys assert param_obj._validate_parameter("find_replace") == ( False, constants.PARAMETER_MSGS["other target env"].format( "_ALL_", param_obj.environment_parameter["find_replace"][0]["replace_value"] ), ) assert param_obj._validate_parameter("spark_pool") == ( False, constants.PARAMETER_MSGS["other target env"].format( "_all_", param_obj.environment_parameter["spark_pool"][0]["replace_value"] ), ) # Overall parameter file should be invalid assert param_obj._validate_parameter_file() == False # Test that mixed environment combinations are invalid for param_dict in param_obj.environment_parameter.get("find_replace"): assert param_obj._validate_environment(param_dict["replace_value"]) == (False, "_ALL_") for param_dict in param_obj.environment_parameter.get("spark_pool"): assert param_obj._validate_environment(param_dict["replace_value"]) == (False, "_all_") finally: # Clean up temporary file Path(temp_file_path).unlink() def test_validate_all_environment_key_with_logging(): """Test that _ALL_ environment key triggers appropriate logging""" import tempfile # Create a temporary parameter file with _ALL_ environment key with tempfile.NamedTemporaryFile(mode="w", suffix=".yml", delete=False) as temp_file: temp_file.write(""" find_replace: - find_value: "test-value" replace_value: _ALL_: "universal-value" spark_pool: - instance_pool_id: "test-pool-id" replace_value: _all_: type: "Capacity" name: "UniversalPool" """) temp_file_path = temp_file.name try: param_obj = Parameter( repository_directory=Path(temp_file_path).parent, item_type_in_scope=["Notebook"], environment="TEST", parameter_file_name=Path(temp_file_path).name, ) # Test environment validation specifically for _ALL_ key replace_value_with_all = {"_all_": "universal-value"} assert param_obj._validate_environment(replace_value_with_all) == (True, "_all_") # Test spark_pool with _ALL_ environment key spark_pool_replace_value_with_all = {"_ALL_": {"type": "Capacity", "name": "UniversalPool"}} assert param_obj._validate_environment(spark_pool_replace_value_with_all) == (True, "_ALL_") # Test environment validation specifically for all key (not reserved) replace_value_with_all = {"all": "universal-value"} assert param_obj._validate_environment(replace_value_with_all) == (False, "env") # Test environment validation with both target env and _ALL_ key (should fail) replace_value_mixed = {"TEST": "test-value", "_ALL_": "universal-value"} assert param_obj._validate_environment(replace_value_mixed) == (False, "_ALL_") # Test spark_pool with mixed environment keys (should fail) spark_pool_mixed = { "TEST": {"type": "Workspace", "name": "TestPool"}, "_ALL_": {"type": "Capacity", "name": "UniversalPool"}, } assert param_obj._validate_environment(spark_pool_mixed) == (False, "_ALL_") # Test environment validation with multiple environment keys including all (not reserved) replace_value_multiple_envs = {"TEST": "test-value", "PROD": "prod-value", "all": "universal-value"} assert param_obj._validate_environment(replace_value_multiple_envs) == (True, "env") # Test spark_pool with multiple environment keys including ALL (not reserved) spark_pool_multiple_envs = { "PROD": {"type": "Workspace", "name": "ProdPool"}, "All": {"type": "Capacity", "name": "UniversalPool"}, } assert param_obj._validate_environment(spark_pool_multiple_envs) == (False, "env") # Test environment validation with only target env (no _ALL_ key) replace_value_target_only = {"TEST": "test-value"} assert param_obj._validate_environment(replace_value_target_only) == (True, "env") # Test environment validation with neither target env nor _ALL_ key replace_value_other = {"PROD": "prod-value"} assert param_obj._validate_environment(replace_value_other) == (False, "env") finally: # Clean up temporary file Path(temp_file_path).unlink() def test_parameter_file_path_absolute(): """Test that Parameter class accepts absolute parameter_file_path.""" import tempfile with tempfile.NamedTemporaryFile(mode="w", suffix=".yml", delete=False) as temp_file: temp_file.write(""" find_replace: - find_value: "test-value" replace_value: TEST: "test-replacement" """) temp_file_path = temp_file.name try: param_obj = Parameter( repository_directory=Path(temp_file_path).parent, item_type_in_scope=["Notebook"], environment="TEST", parameter_file_path=temp_file_path, ) # Should work without errors assert param_obj.environment == "TEST" assert param_obj.item_type_in_scope == ["Notebook"] finally: Path(temp_file_path).unlink() def test_parameter_file_path_relative(): """Test that Parameter class handles relative parameter_file_path by resolving it against repository_directory.""" import tempfile from pathlib import Path # Create a temporary directory to act as the repository with tempfile.TemporaryDirectory() as temp_dir: repo_dir = Path(temp_dir) # Create a nested directory and parameter file relative_dir = "relative/path" (repo_dir / relative_dir).mkdir(parents=True) param_file = "parameters.yml" param_file_path = Path(repo_dir, relative_dir, param_file) param_file_path.write_text("key: value") # Simple valid YAML # Test with relative path that exists relative_path = f"{relative_dir}/{param_file}" # Create a Parameter instance with a relative path param = Parameter( repository_directory=repo_dir, item_type_in_scope=["Notebook"], environment="TEST", parameter_file_path=relative_path, ) # Verify the path was resolved relative to repository_directory expected_path = param_file_path.resolve() assert param.parameter_file_path == expected_path # Test with relative path that doesn't exist non_existent_path = "relative/path/non_existent.yml" # This should not raise an error but should log an error message param2 = Parameter( repository_directory=repo_dir, item_type_in_scope=["Notebook"], environment="TEST", parameter_file_path=non_existent_path, ) # Verify the path was resolved but parameter loading failed assert param2.parameter_file_path is not None assert not param2.environment_parameter # Test with path that exists but is a directory, not a file param3 = Parameter( repository_directory=repo_dir, item_type_in_scope=["Notebook"], environment="TEST", parameter_file_path=relative_dir, ) # Verify the path was resolved but parameter loading failed assert param3.parameter_file_path is not None assert not param3.environment_parameter def test_parameter_file_path_none(): """Test that Parameter class accepts None for parameter_file_path (uses parameter_file_name).""" import tempfile with tempfile.TemporaryDirectory() as temp_dir: temp_dir_path = Path(temp_dir) param_file = temp_dir_path / "parameters.yml" param_file.write_text(""" find_replace: - find_value: "test-value" replace_value: TEST: "test-replacement" """) param_obj = Parameter( repository_directory=temp_dir_path, item_type_in_scope=["Notebook"], environment="TEST", parameter_file_name="parameters.yml", parameter_file_path=None, ) # Should work with parameter_file_name fallback assert param_obj.environment == "TEST" assert param_obj.item_type_in_scope == ["Notebook"] # Verify the parameter_file_path was set correctly from parameter_file_name assert param_obj.parameter_file_path == (temp_dir_path / "parameters.yml").resolve() def test_parameter_file_path_and_name_inputs(): """Test that parameter_file_path takes precedence over parameter_file_name.""" import tempfile with tempfile.TemporaryDirectory() as temp_dir: temp_dir_path = Path(temp_dir) # Create file referenced by parameter_file_name fallback_file = temp_dir_path / "parameters.yml" fallback_file.write_text(""" find_replace: - find_value: "fallback-value" replace_value: TEST: "fallback-replacement" """) # Create file referenced by parameter_file_path primary_file = temp_dir_path / "primary_parameters.yml" primary_file.write_text(""" find_replace: - find_value: "primary-value" replace_value: TEST: "primary-replacement" """) param_obj = Parameter( repository_directory=temp_dir_path, item_type_in_scope=["Notebook"], environment="TEST", parameter_file_name="parameters.yml", # This should be ignored parameter_file_path=str(primary_file), # This should be used ) # Should use primary_file content assert param_obj.environment == "TEST" # Check that the primary file was used by examining parameter content assert "find_replace" in param_obj.environment_parameter assert len(param_obj.environment_parameter["find_replace"]) == 1 assert param_obj.environment_parameter["find_replace"][0]["find_value"] == "primary-value" def test_no_provided_parameter_file_path(): """Test that default behavior without parameter_file_path remains unchanged.""" import tempfile with tempfile.TemporaryDirectory() as temp_dir: temp_dir_path = Path(temp_dir) param_file = temp_dir_path / "parameters.yml" param_file.write_text(""" find_replace: - find_value: "test-value" replace_value: TEST: "test-replacement" """) # Original behavior - no parameter_file_path specified param_obj = Parameter( repository_directory=temp_dir_path, item_type_in_scope=["Notebook"], environment="TEST", parameter_file_name="parameters.yml", ) # Should work exactly as before assert param_obj.environment == "TEST" assert param_obj.item_type_in_scope == ["Notebook"] assert "find_replace" in param_obj.environment_parameter assert len(param_obj.environment_parameter["find_replace"]) == 1 def test_parameter_file_path_nonexistent(): """Test behavior when parameter_file_path points to nonexistent file.""" import tempfile with tempfile.TemporaryDirectory() as temp_dir: nonexistent_path = str(Path(temp_dir) / "nonexistent" / "parameters.yml") # Should log an error but not raise an exception param = Parameter( repository_directory=Path.cwd(), item_type_in_scope=["Notebook"], environment="TEST", parameter_file_path=nonexistent_path, ) # Parameter file path should be set but the environment_parameter should be empty assert param.parameter_file_path is not None assert not param.environment_parameter def test_validate_parameter_file_exists_none(): """Test that _validate_parameter_file_exists returns False when parameter_file_path is None.""" import tempfile with tempfile.TemporaryDirectory() as temp_dir: # Create a Parameter instance with parameter_file_path set to None in _set_parameter_file_path param = Parameter( repository_directory=Path(temp_dir), item_type_in_scope=["Notebook"], environment="TEST", parameter_file_name="does_not_exist.yml", # This file doesn't exist ) # Force parameter_file_path to None param.parameter_file_path = None # Method should return False without raising errors assert param._validate_parameter_file_exists() is False def test_parameter_file_path_invalid_type(): """Test that Parameter class handles invalid types for parameter_file_path.""" import tempfile with tempfile.TemporaryDirectory() as temp_dir: # Parameter class should handle the invalid type internally without raising an exception param = Parameter( repository_directory=Path(temp_dir), item_type_in_scope=["Notebook"], environment="TEST", parameter_file_path=123, # Invalid type ) # The error handling in _set_parameter_file_path sets is_param_path to False # and falls back to the default parameter file path assert param.parameter_file_path is not None assert param.parameter_file_path == (Path(temp_dir) / "parameter.yml").resolve() def test_set_parameter_file_path_error_handling(): """Test error handling in _set_parameter_file_path method.""" import tempfile # Create a mock that raises an exception when called with any arguments path_mock = mock.Mock(side_effect=Exception("Simulated error")) with tempfile.TemporaryDirectory() as temp_dir, mock.patch("fabric_cicd._parameter._parameter.Path", path_mock): # Create parameter with both parameters to test the error handling param = Parameter( repository_directory=temp_dir, # Using string path to avoid early Path conversion item_type_in_scope=["Notebook"], environment="TEST", parameter_file_name="parameters.yml", parameter_file_path="custom_path.yml", ) # The method should have caught the exception and set parameter_file_path to None assert param.parameter_file_path is None def test_basic_template_processing(tmp_path): """Test basic template parameter file processing with valid files.""" # Setup repository structure repo_dir = tmp_path / "repo" repo_dir.mkdir() templates_dir = repo_dir / "templates" templates_dir.mkdir() # Create base parameter file base_file = repo_dir / "parameter.yml" base_content = """ extend: - ./templates/template1.yml find_replace: - find_value: "base-id" replace_value: DEV: "dev-base" PROD: "prod-base" """ base_file.write_text(base_content) # Create template file template_file = templates_dir / "template1.yml" template_content = """ find_replace: - find_value: "template-id" replace_value: DEV: "dev-template" PROD: "prod-template" spark_pool: - instance_pool_id: "pool-id" replace_value: DEV: type: "Workspace" name: "dev-pool" """ template_file.write_text(template_content) # Initialize parameter object param = Parameter(repository_directory=repo_dir, item_type_in_scope=["Notebook"], environment="DEV") # Verify template processing results assert "extend" not in param.environment_parameter assert len(param.environment_parameter["find_replace"]) == 2 assert "spark_pool" in param.environment_parameter # Verify specific values were merged correctly find_values = [item["find_value"] for item in param.environment_parameter["find_replace"]] assert "base-id" in find_values assert "template-id" in find_values def test_missing_templates_directory(tmp_path): """Test handling of missing templates directory.""" # Setup repository without templates directory repo_dir = tmp_path / "repo" repo_dir.mkdir() # Create base parameter file base_file = repo_dir / "parameter.yml" base_content = """ extend: - template1.yml find_replace: - find_value: "base-id" replace_value: DEV: "dev-base" """ base_file.write_text(base_content) # Initialize parameter object param = Parameter(repository_directory=repo_dir, item_type_in_scope=["Notebook"], environment="DEV") # Verify base parameters remain but extend key is removed assert "extend" not in param.environment_parameter assert len(param.environment_parameter["find_replace"]) == 1 assert param.environment_parameter["find_replace"][0]["find_value"] == "base-id" def test_nested_template_prevention(tmp_path): """Test prevention of nested template extensions.""" # Setup repository structure repo_dir = tmp_path / "repo" repo_dir.mkdir() templates_dir = repo_dir / "templates" templates_dir.mkdir() # Create base parameter file base_file = repo_dir / "parameter.yml" base_content = """ extend: - parent.yml find_replace: - find_value: "base-id" replace_value: DEV: "dev-base" """ base_file.write_text(base_content) # Create parent template with nested extend parent_file = templates_dir / "parent.yml" parent_content = """ extend: - child.yml find_replace: - find_value: "parent-id" replace_value: DEV: "dev-parent" """ parent_file.write_text(parent_content) # Create child template child_file = templates_dir / "child.yml" child_content = """ find_replace: - find_value: "child-id" replace_value: DEV: "dev-child" """ child_file.write_text(child_content) # Initialize parameter object param = Parameter(repository_directory=repo_dir, item_type_in_scope=["Notebook"], environment="DEV") # Verify nested template was prevented assert "extend" not in param.environment_parameter # extend key should be removed regardless assert len(param.environment_parameter["find_replace"]) == 1 # only base parameters should be processed find_values = [item["find_value"] for item in param.environment_parameter["find_replace"]] assert "base-id" in find_values assert "parent-id" not in find_values # parent template should be skipped due to nested extend assert "child-id" not in find_values # child template should not be processed def test_template_path_resolution(tmp_path): """Test that template files are resolved relative to the parameter file location.""" # Setup repository structure repo_dir = tmp_path / "repo" repo_dir.mkdir() # Create a directory at the same level as repo for "outside" templates shared_dir = tmp_path / "shared" shared_dir.mkdir() # Create base parameter file base_file = repo_dir / "parameter.yml" base_content = """ extend: - normal.yml # Same directory - ../shared/shared.yml # Outside repo (should work now) - /absolute/path.yml # Absolute path that doesn't exist (should fail) - nonexistent.yml # File doesn't exist (should fail) find_replace: - find_value: "base-id" replace_value: DEV: "dev-base" """ base_file.write_text(base_content) # Create template in same directory normal_file = repo_dir / "normal.yml" normal_content = """ find_replace: - find_value: "normal-id" replace_value: DEV: "dev-normal" """ normal_file.write_text(normal_content) # Create shared template outside repo shared_file = shared_dir / "shared.yml" shared_content = """ find_replace: - find_value: "shared-id" replace_value: DEV: "dev-shared" """ shared_file.write_text(shared_content) # Initialize parameter object param = Parameter(repository_directory=repo_dir, item_type_in_scope=["Notebook"], environment="DEV") # Verify template processing results assert "extend" not in param.environment_parameter # Should have: base, normal, and shared (3 total) assert len(param.environment_parameter["find_replace"]) == 3 find_values = {item["find_value"] for item in param.environment_parameter["find_replace"]} assert find_values == {"base-id", "normal-id", "shared-id"} def test_missing_template_files(tmp_path): """Test that missing template files are handled gracefully.""" repo_dir = tmp_path / "repo" repo_dir.mkdir() base_file = repo_dir / "parameter.yml" base_content = """ extend: - existing.yml - missing.yml - /absolute/missing.yml find_replace: - find_value: "base-id" replace_value: DEV: "dev-base" """ base_file.write_text(base_content) existing_file = repo_dir / "existing.yml" existing_file.write_text(""" find_replace: - find_value: "existing-id" replace_value: DEV: "dev-existing" """) param = Parameter(repository_directory=repo_dir, item_type_in_scope=["Notebook"], environment="DEV") # Only base and existing should be loaded assert len(param.environment_parameter["find_replace"]) == 2 find_values = {item["find_value"] for item in param.environment_parameter["find_replace"]} assert find_values == {"base-id", "existing-id"} def test_template_merge_validation(tmp_path): """Test validation of merged template content.""" # Setup repository structure repo_dir = tmp_path / "repo" repo_dir.mkdir() templates_dir = repo_dir / "templates" templates_dir.mkdir() # Create base parameter file base_file = repo_dir / "parameter.yml" base_content = """ extend: - ./templates/template1.yml - ./templates/invalid.yml find_replace: - find_value: "base-id" replace_value: DEV: "dev-base" """ base_file.write_text(base_content) # Create valid template template1_file = templates_dir / "template1.yml" template1_content = """ find_replace: - find_value: "template-id" replace_value: DEV: "dev-template" """ template1_file.write_text(template1_content) # Create invalid template invalid_file = templates_dir / "invalid.yml" invalid_content = """ find_replace: - replace_value: DEV: "dev-invalid" optional_field: "value" """ invalid_file.write_text(invalid_content) # Initialize parameter object param = Parameter(repository_directory=repo_dir, item_type_in_scope=["Notebook"], environment="DEV") # Verify all content was initially merged assert "extend" not in param.environment_parameter assert len(param.environment_parameter["find_replace"]) == 3 # All entries are merged # Verify the merged content includes both valid and invalid entries (by design) entries = param.environment_parameter["find_replace"] assert any(e.get("find_value") == "base-id" for e in entries) # Base entry assert any(e.get("find_value") == "template-id" for e in entries) # Valid template assert any(e.get("optional_field") == "value" for e in entries) # Invalid template entry # Verify that validation fails due to invalid content is_valid, message = param._validate_parameter("find_replace") assert is_valid == False assert message == constants.PARAMETER_MSGS["missing key"].format("find_replace") assert param._validate_parameter_file() == False def test_template_duplicate_keys_detected(tmp_path): """Test that duplicate keys in template files are detected by _DuplicateKeyLoader.""" repo_dir = tmp_path / "repo" repo_dir.mkdir() templates_dir = repo_dir / "templates" templates_dir.mkdir() # Create base parameter file base_file = repo_dir / "parameter.yml" base_content = """ extend: - ./templates/duplicate_template.yml find_replace: - find_value: "base-id" replace_value: DEV: "dev-base" """ base_file.write_text(base_content) # Create template file with duplicate keys template_file = templates_dir / "duplicate_template.yml" template_content = """ find_replace: - find_value: "template-id" replace_value: DEV: "dev-template" find_replace: - find_value: "duplicate-template-id" replace_value: DEV: "dev-duplicate" """ template_file.write_text(template_content) # Initialize parameter object param = Parameter(repository_directory=repo_dir, item_type_in_scope=["Notebook"], environment="DEV") # Template with duplicate keys should be skipped, only base content should remain assert "extend" not in param.environment_parameter assert len(param.environment_parameter["find_replace"]) == 1 assert param.environment_parameter["find_replace"][0]["find_value"] == "base-id" def test_circular_template_reference(tmp_path): """Test handling of circular template references.""" repo_dir = tmp_path / "repo" repo_dir.mkdir() templates_dir = repo_dir / "templates" templates_dir.mkdir() # Create base parameter file that references template1 base_file = repo_dir / "parameter.yml" base_content = """ extend: - template1.yml find_replace: - find_value: "base-id" replace_value: DEV: "dev-base" """ base_file.write_text(base_content) # Create template1 that references template2 template1_file = templates_dir / "template1.yml" template1_content = """ extend: - template2.yml find_replace: - find_value: "template1-id" replace_value: DEV: "dev-template1" """ template1_file.write_text(template1_content) # Create template2 that references template1 (circular) template2_file = templates_dir / "template2.yml" template2_content = """ extend: - template1.yml find_replace: - find_value: "template2-id" replace_value: DEV: "dev-template2" """ template2_file.write_text(template2_content) # Initialize parameter object param = Parameter(repository_directory=repo_dir, item_type_in_scope=["Notebook"], environment="DEV") # Verify only base content remains due to circular reference detection assert "extend" not in param.environment_parameter assert len(param.environment_parameter["find_replace"]) == 1 assert param.environment_parameter["find_replace"][0]["find_value"] == "base-id" def test_multiple_template_references(tmp_path): """Test handling of multiple template references with various scenarios.""" repo_dir = tmp_path / "repo" repo_dir.mkdir() templates_dir = repo_dir / "templates" templates_dir.mkdir() # Create different types of template files template_configs = [ # Basic template with single find_replace ( "template1.yml", """ find_replace: - find_value: "template1-id" replace_value: DEV: "dev-template1" PROD: "prod-template1" """, ), # Template with multiple find_replace entries ( "template2.yml", """ find_replace: - find_value: "template2-id1" replace_value: DEV: "dev-template2-1" - find_value: "template2-id2" replace_value: DEV: "dev-template2-2" """, ), # Template with regex and item filters ( "template3.yml", """ find_replace: - find_value: "template3-.*" is_regex: "true" item_type: "Notebook" item_name: "Test Notebook" replace_value: DEV: "dev-template3" """, ), # Template with _ALL_ environment ( "template4.yml", """ find_replace: - find_value: "template4-id" replace_value: _ALL_: "all-template4" """, ), # Template with key_value_replace ( "template5.yml", """ key_value_replace: - find_key: "connectionString" replace_value: DEV: "dev-connection" PROD: "prod-connection" """, ), ] # Create template files template_refs = [] for template_name, content in template_configs: template_refs.append(template_name) template_file = templates_dir / template_name template_file.write_text(content.strip(), encoding="utf-8") # Create base parameter file base_file = repo_dir / "parameter.yml" template_refs_with_path = [f"./templates/{ref}" for ref in template_refs] base_content = """ find_replace: - find_value: "base-id" replace_value: DEV: "dev-base" PROD: "prod-base" - find_value: "base-regex-.*" is_regex: "true" replace_value: DEV: "dev-base-regex" key_value_replace: - find_key: "baseKey" replace_value: DEV: "dev-base-value" extend: """ + yaml.safe_dump(template_refs_with_path, allow_unicode=True, indent=2) base_file.write_text(base_content.strip(), encoding="utf-8") # Create a test notebook item for validation notebook_dir = repo_dir / "TestNotebook" notebook_dir.mkdir() platform_file = notebook_dir / ".platform" platform_content = {"metadata": {"type": "Notebook", "displayName": "Test Notebook"}} platform_file.write_text(json.dumps(platform_content), encoding="utf-8") # Test with DEV environment param_dev = Parameter(repository_directory=repo_dir, item_type_in_scope=["Notebook"], environment="DEV") # Validate basic file operations assert param_dev._validate_parameter_file_exists(), "Parameter file does not exist" is_valid, message = param_dev._validate_load_parameters_to_dict() assert is_valid, f"Failed to load parameters: {message}" # Validate merged parameters find_replace_params = param_dev.environment_parameter["find_replace"] key_value_params = param_dev.environment_parameter.get("key_value_replace", []) # Test base parameter presence assert any(p["find_value"] == "base-id" for p in find_replace_params), "Base find_replace missing" assert any(p["find_value"] == "base-regex-.*" for p in find_replace_params), "Base regex find_replace missing" assert any(p["find_key"] == "baseKey" for p in key_value_params), "Base key_value_replace missing" # Test template merging assert any(p["find_value"] == "template1-id" for p in find_replace_params), "Template1 not merged" assert any(p["find_value"] == "template2-id1" for p in find_replace_params), "Template2 first entry not merged" assert any(p["find_value"] == "template2-id2" for p in find_replace_params), "Template2 second entry not merged" assert any(p["find_value"] == "template3-.*" for p in find_replace_params), "Template3 regex not merged" assert any(p["find_value"] == "template4-id" for p in find_replace_params), "Template4 _ALL_ not merged" assert any(p["find_key"] == "connectionString" for p in key_value_params), "Template5 key_value_replace not merged" # Test regex validation regex_entries = [p for p in find_replace_params if p.get("is_regex") == "true"] assert len(regex_entries) == 2, "Expected exactly 2 regex entries" for entry in regex_entries: assert re.compile(entry["find_value"]), f"Invalid regex pattern: {entry['find_value']}" # Test item filtering filtered_entries = [p for p in find_replace_params if p.get("item_name") or p.get("item_type")] assert len(filtered_entries) == 1, "Expected exactly 1 filtered entry" filtered_entry = filtered_entries[0] assert filtered_entry["item_type"] == "Notebook", "Incorrect item type filter" assert filtered_entry["item_name"] == "Test Notebook", "Incorrect item name filter" # Test _ALL_ environment handling all_env_entries = [p for p in find_replace_params if "_ALL_" in p["replace_value"]] assert len(all_env_entries) == 1, "Expected exactly 1 _ALL_ environment entry" assert all_env_entries[0]["replace_value"]["_ALL_"] == "all-template4", "Incorrect _ALL_ value" # Test with PROD environment param_prod = Parameter(repository_directory=repo_dir, item_type_in_scope=["Notebook"], environment="PROD") assert param_prod._validate_parameter_file_exists(), "Parameter file does not exist for PROD" is_valid, message = param_prod._validate_parameter("find_replace") assert is_valid, f"Parameter validation failed for PROD: {message}" # Verify environment-specific values prod_params = param_prod.environment_parameter["find_replace"] assert any( p["find_value"] == "template1-id" and p["replace_value"]["PROD"] == "prod-template1" for p in prod_params ), "PROD environment value not correctly loaded" def test_template_merge_behavior(tmp_path): """Test template merging behavior including order, duplicates, and identical entries.""" repo_dir = tmp_path / "repo" repo_dir.mkdir() templates_dir = repo_dir / "templates" templates_dir.mkdir() # Create base parameter file base_file = repo_dir / "parameter.yml" base_content = """ extend: - ./templates/template1.yml - ./templates/template1.yml # Duplicate reference - ./templates/template2.yml find_replace: - find_value: "id-1" replace_value: DEV: "base-1" PROD: "base-2" """ base_file.write_text(base_content) # Create template1 with identical and different entries template1_file = templates_dir / "template1.yml" template1_content = """ find_replace: - find_value: "id-1" # Identical to base replace_value: DEV: "base-1" PROD: "base-2" - find_value: "id-2" # Unique entry replace_value: DEV: "template1-1" PROD: "template1-2" """ template1_file.write_text(template1_content) # Create template2 with different values template2_file = templates_dir / "template2.yml" template2_content = """ find_replace: - find_value: "id-1" # Different values replace_value: DEV: "template2-1" PROD: "template2-2" """ template2_file.write_text(template2_content) # Initialize parameter object param = Parameter(repository_directory=repo_dir, item_type_in_scope=["Notebook"], environment="DEV") # Test 1: Template deduplication find_values = {item["find_value"] for item in param.environment_parameter["find_replace"]} assert len(find_values) == 2, "Duplicate template references should be processed only once" # Test 2: Merge order preservation find_replace_entries = param.environment_parameter["find_replace"] assert find_replace_entries[0]["find_value"] == "id-1" # Base entry first assert find_replace_entries[-1]["find_value"] == "id-1" # Template2 entry last # Test 3: Value preservation dev_values = {item["replace_value"]["DEV"] for item in find_replace_entries if item["find_value"] == "id-1"} assert "base-1" in dev_values, "Base values should be preserved" assert "template2-1" in dev_values, "Template values should be preserved" def test_template_reference_handling(tmp_path): """Test template reference handling including circular references and deep nesting.""" repo_dir = tmp_path / "repo" repo_dir.mkdir() templates_dir = repo_dir / "templates" templates_dir.mkdir() # Create base file base_file = repo_dir / "parameter.yml" base_content = """ extend: - circular1.yml - deep1.yml find_replace: - find_value: "base-id" replace_value: DEV: "dev-base" """ base_file.write_text(base_content) # Create circular reference templates circular1_content = """ extend: - circular2.yml find_replace: - find_value: "circular1-id" replace_value: DEV: "dev-circular1" """ (templates_dir / "circular1.yml").write_text(circular1_content) circular2_content = """ extend: - circular1.yml find_replace: - find_value: "circular2-id" replace_value: DEV: "dev-circular2" """ (templates_dir / "circular2.yml").write_text(circular2_content) # Create deep nesting templates for i in range(1, 6): template_content = f""" extend: - deep{i + 1}.yml find_replace: - find_value: "deep{i}-id" replace_value: DEV: "dev-deep{i}" """ (templates_dir / f"deep{i}.yml").write_text(template_content) # Create final template in deep chain final_content = """ find_replace: - find_value: "deep6-id" replace_value: DEV: "dev-deep6" """ (templates_dir / "deep6.yml").write_text(final_content) # Initialize parameter object param = Parameter(repository_directory=repo_dir, item_type_in_scope=["Notebook"], environment="DEV") # Test 1: Circular reference handling circular_values = { item["find_value"] for item in param.environment_parameter["find_replace"] if "circular" in item["find_value"] } assert not circular_values, "Circular references should be prevented" # Test 2: Deep nesting limit deep_entries = [item for item in param.environment_parameter["find_replace"] if "deep" in item["find_value"]] assert len(deep_entries) <= 5, "Deep nesting should be limited" # Test 3: Base content preservation assert any(item["find_value"] == "base-id" for item in param.environment_parameter["find_replace"]), ( "Base content should be preserved" ) @pytest.fixture def empty_parameter(tmp_path): # Parameter expects a repository directory; use an empty temporary path. return Parameter(repository_directory=tmp_path, item_type_in_scope=["Notebook"], environment="DEV") def test_validate_key_value_find_key_valid_dot_notation(empty_parameter): param = {"find_key": "$.server.host"} ok, msg = empty_parameter._validate_key_value_find_key(param) assert ok is True assert msg == "Valid JSONPath" def test_validate_key_value_find_key_valid_filter_syntax(empty_parameter): param = {"find_key": '$.variables[?(@.name=="SQL_Server")].value'} ok, msg = empty_parameter._validate_key_value_find_key(param) assert ok is True assert msg == "Valid JSONPath" def test_validate_key_value_find_key_missing_key(empty_parameter): param = {} ok, msg = empty_parameter._validate_key_value_find_key(param) assert ok is False assert "Missing or empty 'find_key'" in msg def test_validate_key_value_find_key_non_string(empty_parameter): param = {"find_key": 123} ok, msg = empty_parameter._validate_key_value_find_key(param) assert ok is False assert "Missing or empty 'find_key'" in msg def test_validate_key_value_find_key_empty_string(empty_parameter): param = {"find_key": ""} ok, msg = empty_parameter._validate_key_value_find_key(param) assert ok is False assert "Missing or empty 'find_key'" in msg def test_validate_key_value_find_key_requires_root(empty_parameter): param = {"find_key": 'variables[?(@.name=="SQL_Server")].value'} ok, msg = empty_parameter._validate_key_value_find_key(param) assert ok is False assert "must be an absolute JSONPath" in msg def test_validate_key_value_find_key_unbalanced_filter(empty_parameter): param = {"find_key": '$.variables[?(@.name=="SQL_Server"].value'} ok, msg = empty_parameter._validate_key_value_find_key(param) assert ok is False assert "Invalid JSONPath expression" in msg def test_validate_key_value_find_key_unsupported_regex_operator(empty_parameter): # expressions using =~ are commonly unsupported by jsonpath_ng; ensure validator rejects them param = {"find_key": "$.variables[?(@.name =~ /SQL_.*/)].value"} ok, msg = empty_parameter._validate_key_value_find_key(param) assert ok is False assert "Invalid JSONPath expression" in msg def test_validate_required_values_integration_calls_find_key_validator(empty_parameter): # Integration: ensure _validate_required_values uses the find_key validator for key_value_replace param_dict = {"find_key": "no-root", "replace_value": {"DEV": "x"}} ok, msg = empty_parameter._validate_required_values("key_value_replace", param_dict) assert ok is False assert "must be an absolute JSONPath" in msg def test_validate_and_evaluate_bracket_key_with_yaml(empty_parameter): """JSONPath with bracket notation should parse and match YAML keys with spaces.""" import yaml from jsonpath_ng.ext import parse yaml_str = """ "my key": value: 42 """ param = {"find_key": '$["my key"].value'} ok, msg = empty_parameter._validate_key_value_find_key(param) assert ok is True assert msg == "Valid JSONPath" data = yaml.safe_load(yaml_str) matches = parse(param["find_key"]).find(data) assert len(matches) == 1 assert matches[0].value == 42 def test_yaml_boolean_filter_evaluation(empty_parameter): """JSONPath filter on boolean YAML scalars should evaluate correctly.""" import yaml from jsonpath_ng.ext import parse yaml_str = """ servers: - name: "a" enabled: true - name: "b" enabled: false """ param = {"find_key": "$.servers[?(@.enabled==true)].name"} ok, msg = empty_parameter._validate_key_value_find_key(param) assert ok is True assert msg == "Valid JSONPath" data = yaml.safe_load(yaml_str) matches = parse(param["find_key"]).find(data) # Expect exactly one matching server name ("a") assert len(matches) == 1 assert matches[0].value == "a" def test_yaml_no_match_is_no_op(empty_parameter): """A JSONPath that matches nothing in YAML should be a no-op (no exception).""" import yaml from jsonpath_ng.ext import parse yaml_str = """ config: flag: false """ param = {"find_key": "$.config.nonexistent"} ok, msg = empty_parameter._validate_key_value_find_key(param) assert ok is True assert msg == "Valid JSONPath" data = yaml.safe_load(yaml_str) matches = parse(param["find_key"]).find(data) assert len(matches) == 0 @pytest.mark.parametrize( ("param_value", "expected_ok", "expected_msg_contains"), [ # ===== Legacy format tests ===== pytest.param( [{"connection_id": "76e05dfe-9855-4e3d-a410-1dda048dbe99", "semantic_model_name": ["model1", "model2"]}], True, "parameter is valid", id="legacy_string_connection_id", ), pytest.param( [ { "connection_id": { "PPE": "76e05dfe-9855-4e3d-a410-1dda048dbe99", "PROD": "a1b2c3d4-5678-90ab-cdef-1234567890ab", }, "semantic_model_name": ["model1", "model2"], } ], False, "must be a string guid", id="legacy_dict_connection_id_not_supported", ), pytest.param( [ { "connection_id": "invalid-guid-format", "semantic_model_name": ["model1"], } ], False, "not a valid guid", id="legacy_invalid_guid", ), pytest.param( [ { "connection_id": 12345, "semantic_model_name": ["model1"], } ], False, "must be a string guid", id="legacy_connection_id_not_string", ), pytest.param( [{"connection_id": "", "semantic_model_name": ["model1"]}], False, "missing value", id="legacy_empty_connection_id", ), pytest.param( [ { "connection_id": "76e05dfe-9855-4e3d-a410-1dda048dbe99", "semantic_model_name": "Model1", "default": "x", } ], False, "mixed format", id="legacy_mixed_with_new_keys", ), # ===== New format tests ===== pytest.param( {"default": {"connection_id": {"DEV": "76e05dfe-9855-4e3d-a410-1dda048dbe99"}}}, True, "parameter is valid", id="new_default_only", ), pytest.param( { "models": [ {"semantic_model_name": "MyModel", "connection_id": {"DEV": "76e05dfe-9855-4e3d-a410-1dda048dbe99"}} ] }, True, "parameter is valid", id="new_models_only", ), pytest.param( { "default": {"connection_id": {"DEV": "76e05dfe-9855-4e3d-a410-1dda048dbe99"}}, "models": [ { "semantic_model_name": ["Model1", "Model2"], "connection_id": { "DEV": "a1b2c3d4-5678-90ab-cdef-1234567890ab", "PROD": "b2c3d4e5-6789-0abc-def1-234567890abc", }, } ], }, True, "parameter is valid", id="new_default_and_models", ), pytest.param( {}, False, "requires 'default' or 'models'", id="new_empty_dict", ), pytest.param( {"default": {"connection_id": {"DEV": "76e05dfe-9855-4e3d-a410-1dda048dbe99"}}, "invalid_key": "x"}, False, "invalid key", id="new_invalid_key", ), pytest.param( {"default": "not-a-dict"}, False, "dictionary", id="new_default_not_dict", ), pytest.param( {"default": {"some_key": "value"}}, False, "connection_id", id="new_default_missing_connection_id", ), pytest.param( {"models": []}, False, "non-empty list", id="new_models_empty_list", ), pytest.param( {"models": "not-a-list"}, False, "non-empty list", id="new_models_not_list", ), pytest.param( {"models": ["not-a-dict"]}, False, "dictionary", id="new_model_entry_not_dict", ), pytest.param( {"models": [{"connection_id": {"DEV": "76e05dfe-9855-4e3d-a410-1dda048dbe99"}}]}, False, "semantic_model_name", id="new_missing_semantic_model_name", ), pytest.param( {"models": [{"semantic_model_name": "MyModel"}]}, False, "connection_id", id="new_missing_connection_id", ), pytest.param( { "models": [ {"semantic_model_name": 12345, "connection_id": {"DEV": "76e05dfe-9855-4e3d-a410-1dda048dbe99"}} ] }, False, "string or list[string]", id="new_invalid_semantic_model_name_type", ), pytest.param( {"models": [{"semantic_model_name": "MyModel", "connection_id": "76e05dfe-9855-4e3d-a410-1dda048dbe99"}]}, False, "must be a dictionary", id="new_string_connection_id_not_supported", ), pytest.param( {"models": [{"semantic_model_name": "MyModel", "connection_id": {"DEV": "invalid-guid"}}]}, False, "not a valid guid", id="new_invalid_connection_guid", ), pytest.param( {"models": [{"semantic_model_name": "MyModel", "connection_id": {}}]}, False, "non-empty dictionary", id="new_empty_connection_id_dict", ), ], ) def test_semantic_model_binding_validation(empty_parameter, param_value, expected_ok, expected_msg_contains): """Parametrized test for semantic_model_binding validation covering legacy and new formats.""" empty_parameter.environment_parameter = {"semantic_model_binding": param_value} ok, msg = empty_parameter._validate_semantic_model_binding_parameter("semantic_model_binding") assert ok is expected_ok assert expected_msg_contains.lower() in msg.lower() @pytest.mark.parametrize( ("connection_id", "require_string", "require_dict", "expected_ok", "expected_msg_contains"), [ pytest.param( "76e05dfe-9855-4e3d-a410-1dda048dbe99", True, False, True, "Valid", id="legacy_valid_string_guid", ), pytest.param( "invalid-guid", True, False, False, "not a valid GUID", id="legacy_invalid_guid", ), pytest.param( {"DEV": "76e05dfe-9855-4e3d-a410-1dda048dbe99"}, True, False, False, "must be a string GUID", id="legacy_dict_not_supported", ), pytest.param( { "PPE": "76e05dfe-9855-4e3d-a410-1dda048dbe99", "PROD": "a1b2c3d4-5678-90ab-cdef-1234567890ab", }, False, True, True, "Valid", id="new_valid_multi_env", ), pytest.param( {"PPE": "not-a-guid", "PROD": "a1b2c3d4-5678-90ab-cdef-1234567890ab"}, False, True, False, "not a valid GUID", id="new_invalid_guid_format", ), pytest.param( "76e05dfe-9855-4e3d-a410-1dda048dbe99", False, True, False, "must be a dictionary", id="new_string_not_supported", ), ], ) def test_validate_connection_id( empty_parameter, connection_id, require_string, require_dict, expected_ok, expected_msg_contains ): """Test _validate_connection_id with various inputs.""" ok, msg = empty_parameter._validate_connection_id( connection_id, "semantic_model_binding", require_string=require_string, require_dict=require_dict ) assert ok is expected_ok assert expected_msg_contains in msg def test_semantic_model_binding_new_format_models_invalid_connection_guid(empty_parameter): """Test semantic_model_binding new format with invalid GUID in models connections.""" empty_parameter.environment_parameter = { "semantic_model_binding": { "models": [{"semantic_model_name": "MyModel", "connection_id": {"DEV": "not-a-valid-guid"}}] } } ok, msg = empty_parameter._validate_semantic_model_binding_parameter("semantic_model_binding") assert ok is False assert "not a valid guid" in msg.lower() def test_semantic_model_binding_legacy_format_mixed_with_new_keys(empty_parameter): """Test semantic_model_binding legacy format with new format keys mixed in (should fail).""" empty_parameter.environment_parameter = { "semantic_model_binding": [ { "connection_id": "76e05dfe-9855-4e3d-a410-1dda048dbe99", "semantic_model_name": "Model1", "default": "should-not-be-here", # New format key in legacy entry } ] } ok, msg = empty_parameter._validate_semantic_model_binding_parameter("semantic_model_binding") assert ok is False assert "mixed format" in msg.lower() @pytest.mark.parametrize( ("param_value", "is_new_format", "expected_duplicates"), [ # New format: duplicate name triggers warning ( { "default": {"connection_id": {"PPE": "00000000-0000-0000-0000-000000000001"}}, "models": [ {"semantic_model_name": "ModelA", "connection_id": {"PPE": "00000000-0000-0000-0000-000000000002"}}, {"semantic_model_name": "ModelA", "connection_id": {"PPE": "00000000-0000-0000-0000-000000000003"}}, ], }, True, {"ModelA"}, ), # New format: no duplicates, no warning ( { "default": {"connection_id": {"PPE": "00000000-0000-0000-0000-000000000001"}}, "models": [ {"semantic_model_name": "ModelA", "connection_id": {"PPE": "00000000-0000-0000-0000-000000000002"}}, {"semantic_model_name": "ModelB", "connection_id": {"PPE": "00000000-0000-0000-0000-000000000003"}}, ], }, True, set(), ), # Legacy format: duplicate name triggers warning ( [ {"semantic_model_name": "ModelA", "connection_id": "00000000-0000-0000-0000-000000000001"}, {"semantic_model_name": "ModelA", "connection_id": "00000000-0000-0000-0000-000000000002"}, ], False, {"ModelA"}, ), # Legacy format: no duplicates, no warning ( [ {"semantic_model_name": "ModelA", "connection_id": "00000000-0000-0000-0000-000000000001"}, {"semantic_model_name": "ModelB", "connection_id": "00000000-0000-0000-0000-000000000002"}, ], False, set(), ), # New format: duplicate within a list value ( { "default": {"connection_id": {"PPE": "00000000-0000-0000-0000-000000000001"}}, "models": [ { "semantic_model_name": ["ModelA", "ModelA"], "connection_id": {"PPE": "00000000-0000-0000-0000-000000000002"}, }, ], }, True, {"ModelA"}, ), # Legacy format: duplicate across list values in different entries ( [ {"semantic_model_name": ["ModelA", "ModelB"], "connection_id": "00000000-0000-0000-0000-000000000001"}, {"semantic_model_name": ["ModelB", "ModelC"], "connection_id": "00000000-0000-0000-0000-000000000002"}, ], False, {"ModelB"}, ), ], ids=[ "new_format_duplicate", "new_format_no_duplicate", "legacy_duplicate", "legacy_no_duplicate", "new_format_duplicate_within_list", "legacy_duplicate_across_lists", ], ) def test_check_duplicate_semantic_model_names(empty_parameter, param_value, is_new_format, expected_duplicates, caplog): """Test that _check_duplicate_semantic_model_names warns on duplicate names.""" import logging with caplog.at_level(logging.WARNING): empty_parameter._check_duplicate_semantic_model_names(param_value, is_new_format) if expected_duplicates: expected_msg = constants.PARAMETER_MSGS["duplicate_semantic_model"].format( ", ".join(sorted(expected_duplicates)) ) assert expected_msg in caplog.messages, ( f"Expected warning message not found.\nExpected: {expected_msg}\nActual messages: {caplog.messages}" ) # Verify exactly one warning was logged duplicate_warnings = [m for m in caplog.messages if "Duplicate semantic model names found" in m] assert len(duplicate_warnings) == 1, ( f"Expected exactly 1 duplicate warning, found {len(duplicate_warnings)}: {duplicate_warnings}" ) # Verify duplicates produce a warning but do not cause validation failure empty_parameter.environment_parameter = {"semantic_model_binding": param_value} ok, _ = empty_parameter._validate_semantic_model_binding_parameter("semantic_model_binding") assert ok is True, "Duplicate semantic model names should warn but not fail validation" else: assert not any("Duplicate semantic model names found" in m for m in caplog.messages), ( f"Unexpected duplicate warning found in messages: {caplog.messages}" ) ================================================ FILE: tests/test_parameter_utils.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """ Tests for the parameter utility functions in _utils.py. The tests focused on path handling functions should be compatible with both Windows and Linux. """ import json import logging import re import shutil import tempfile from pathlib import Path from unittest import mock from unittest.mock import MagicMock import pytest import yaml import fabric_cicd.constants as constants # Logger for testing logger = logging.getLogger(__name__) @pytest.fixture def temp_repository(): """Creates a temporary directory structure mocking a repository for testing.""" temp_dir = Path(tempfile.mkdtemp()) try: # Create test directory structure (temp_dir / "folder1").mkdir() (temp_dir / "folder1" / "subfolder").mkdir() (temp_dir / "folder2").mkdir() # Create test files (temp_dir / "file1.txt").write_text("content1") (temp_dir / "file2.json").write_text("content2") (temp_dir / "folder1" / "file3.py").write_text("content3") (temp_dir / "folder1" / "subfolder" / "file4.md").write_text("content4") (temp_dir / "folder2" / "file5.txt").write_text("content5") # Return the temporary directory path yield temp_dir finally: # Clean up temporary directory after tests shutil.rmtree(temp_dir) from fabric_cicd._common._exceptions import InputError, ParsingError from fabric_cicd._parameter._utils import ( _check_parameter_structure, _extract_item_attribute, _find_match, _process_regular_path, _process_wildcard_path, _resolve_file_path, _validate_wildcard_syntax, check_replacement, extract_find_value, extract_parameter_filters, extract_replace_value, is_valid_structure, process_environment_key, process_input_path, replace_key_value, replace_variables_in_parameter_file, ) class TestParameterUtilities: """Tests for parameter utilities in _utils.py.""" @pytest.fixture def mock_workspace(self): """Creates a mock FabricWorkspace for testing.""" mock_ws = mock.MagicMock() mock_ws.repository_directory = Path("/mock/repository") mock_ws.workspace_id = "mock-workspace-id" mock_ws.workspace_items = { "Notebook": { "Test Notebook": {"id": "notebook-id", "sqlendpoint": "", "sqlendpointid": "", "queryserviceuri": ""}, }, "Warehouse": { "TestWarehouse": { "id": "warehouse-id", "sqlendpoint": "warehouse-endpoint", "sqlendpointid": "", "queryserviceuri": "", }, }, "Lakehouse": { "Test_Lakehouse": { "id": "lakehouse-id", "sqlendpoint": "lakehouse-endpoint", "sqlendpointid": "lakehouse-sql-endpoint-id", "queryserviceuri": "", }, }, "Eventhouse": { "Test Eventhouse": { "id": "eventhouse-id", "sqlendpoint": "", "sqlendpointid": "", "queryserviceuri": "eventhouse-query-uri", }, }, "SQLDatabase": { "TestSQLDatabase": { "id": "sqldatabase-id", "sqlendpoint": "test-sql-server.database.fabric.microsoft.com,1433", "sqlendpointid": "", "queryserviceuri": "", }, }, } mock_ws.repository_items = { "Dataflow": { "Source Dataflow": {"id": "source-dataflow-id"}, } } # Mock _refresh_deployed_items to avoid API calls in all tests using this fixture mock_ws._refresh_deployed_items = MagicMock() return mock_ws def test_extract_find_value(self): """Tests extract_find_value with string.""" # Test with plain text param_dict = {"find_value": "test-value"} expected = {"pattern": "test-value", "is_regex": False, "has_matches": True} assert extract_find_value(param_dict, "content with test-value", True) == expected expected_no_match = {"pattern": "test-value", "is_regex": False, "has_matches": False} assert extract_find_value(param_dict, "unrelated content", True) == expected_no_match def test_extract_find_value_valid_regex(self): """Tests extract_find_value with regex pattern.""" param_dict = {"find_value": "id=([\\w-]+)", "is_regex": "true"} # Test with regex expected = {"pattern": "id=([\\w-]+)", "is_regex": True, "has_matches": True} assert extract_find_value(param_dict, "content with id=abc-123", True) == expected # Test with non-matching regex expected_no_match = {"pattern": "id=([\\w-]+)", "is_regex": True, "has_matches": False} assert extract_find_value(param_dict, "unrelated content", True) == expected_no_match # Test with regex but filter_match=False - should still return is_regex=True expected = {"pattern": "id=([\\w-]+)", "is_regex": True, "has_matches": False} assert extract_find_value(param_dict, "content with id=abc-123", False) == expected def test_extract_find_value_invalid_regex(self): """Tests extract_find_value with invalid regex capturing groups.""" # Test with regex that has no capturing groups param_dict = {"find_value": "id=\\w+", "is_regex": "true"} with pytest.raises(InputError): extract_find_value(param_dict, "content with id=abc123", True) # Test with regex that has multiple capturing groups param_dict = {"find_value": "(id)=([\\w-]+)", "is_regex": "true"} with pytest.raises(InputError): extract_find_value(param_dict, "content with id=abc-123", True) # Test with regex that captures empty value param_dict = {"find_value": "id=()", "is_regex": "true"} with pytest.raises(InputError): extract_find_value(param_dict, "content with id=", True) # Test that structure validation happens even when there are no matches # (regex with no capturing groups should still fail even if no matches) param_dict = {"find_value": "id=\\w+", "is_regex": "true"} with pytest.raises(InputError): extract_find_value(param_dict, "unrelated content without matches", True) # Test that structure validation happens with filter_match=False too param_dict = {"find_value": "(id)=([\\w-]+)", "is_regex": "true"} with pytest.raises(InputError): extract_find_value(param_dict, "unrelated content without matches", False) def test_extract_find_value_multiple_matches(self): """Tests extract_find_value with regex pattern that has multiple matches.""" param_dict = {"find_value": "id=([\\w-]+)", "is_regex": "true"} # Test with multiple matches in content - now returns simple dict format content_with_multiple = "content with id=abc-123 and id=def-456 and id=ghi-789" result = extract_find_value(param_dict, content_with_multiple, True) expected = {"pattern": "id=([\\w-]+)", "is_regex": True, "has_matches": True} assert result == expected # Test that content is detected as having matches content_with_duplicates = "content with id=abc-123 and id=def-456 and id=abc-123" result = extract_find_value(param_dict, content_with_duplicates, True) expected = {"pattern": "id=([\\w-]+)", "is_regex": True, "has_matches": True} assert result == expected def test_extract_replace_value_default(self, mock_workspace): """Tests extract_replace_value with different inputs, get_dataflow_name=False.""" # Regular string should be returned as is assert extract_replace_value(mock_workspace, "literal string") == "literal string" # Workspace ID variable should return the workspace ID assert extract_replace_value(mock_workspace, "$workspace.id", False) == "mock-workspace-id" # Workspace ID variable should return the workspace ID assert extract_replace_value(mock_workspace, "$workspace.$id", False) == "mock-workspace-id" # Workspace name variable should resolve to workspace ID with mock.patch("fabric_cicd._parameter._utils._extract_workspace_id") as mock_extract_ws: mock_extract_ws.return_value = "resolved-workspace-id" result = extract_replace_value(mock_workspace, "$workspace.dev") assert result == "resolved-workspace-id" mock_extract_ws.assert_called_once_with(mock_workspace, "$workspace.dev") # Item attribute variables should extract values from workspace items with mock.patch("fabric_cicd._parameter._utils._extract_item_attribute") as mock_extract: mock_extract.return_value = "notebook-id" result = extract_replace_value(mock_workspace, "$items.Notebook.Test Notebook.id") assert result == "notebook-id" mock_extract.assert_called_once_with(mock_workspace, "$items.Notebook.Test Notebook.id", False) def test_extract_replace_value_get_dataflow_name(self, mock_workspace): """Tests extract_replace_value with different inputs, get_dataflow_name=True.""" # With get_dataflow_name=True for regular string, should return None assert extract_replace_value(mock_workspace, "literal string", True) is None # With get_dataflow_name=True for workspace ID, should return an error with pytest.raises( InputError, match=re.escape( "Invalid replace_value variable: '$workspace'. Expected format to get dataflow name: $items.type.name.$attribute" ), ): result = extract_replace_value(mock_workspace, "$workspace.id", True) # With get_dataflow_name=True for non-Dataflow item, should return None with mock.patch("fabric_cicd._parameter._utils._extract_item_attribute") as mock_extract: mock_extract.return_value = None result = extract_replace_value(mock_workspace, "$items.Notebook.Test Notebook.id", True) assert result is None mock_extract.assert_called_once_with(mock_workspace, "$items.Notebook.Test Notebook.id", True) # With get_dataflow_name=True for a Dataflow item, should return the Dataflow name with mock.patch("fabric_cicd._parameter._utils._extract_item_attribute") as mock_extract: mock_extract.return_value = "Source Dataflow" result = extract_replace_value(mock_workspace, "$items.Dataflow.Source Dataflow.id", True) assert result == "Source Dataflow" mock_extract.assert_called_once_with(mock_workspace, "$items.Dataflow.Source Dataflow.id", True) def test_extract_item_attribute_valid(self, mock_workspace): """Tests _extract_item_attribute with valid variables.""" # Test with valid notebook item result = _extract_item_attribute(mock_workspace, "$items.Notebook.Test Notebook.id", False) assert result == "notebook-id" result = _extract_item_attribute(mock_workspace, "$items.Notebook.Test Notebook.$id", False) assert result == "notebook-id" # Test with valid lakehouse item result = _extract_item_attribute(mock_workspace, "$items.Lakehouse.Test_Lakehouse.sqlendpoint", False) assert result == "lakehouse-endpoint" result = _extract_item_attribute(mock_workspace, "$items.Lakehouse.Test_Lakehouse.$sqlendpoint", False) assert result == "lakehouse-endpoint" # Test with valid lakehouse sqlendpointid attribute result = _extract_item_attribute(mock_workspace, "$items.Lakehouse.Test_Lakehouse.sqlendpointid", False) assert result == "lakehouse-sql-endpoint-id" result = _extract_item_attribute(mock_workspace, "$items.Lakehouse.Test_Lakehouse.$sqlendpointid", False) assert result == "lakehouse-sql-endpoint-id" # Test with valid warehouse item result = _extract_item_attribute(mock_workspace, "$items.Warehouse.TestWarehouse.id", False) assert result == "warehouse-id" result = _extract_item_attribute(mock_workspace, "$items.Warehouse.TestWarehouse.$id", False) assert result == "warehouse-id" # Test with valid SQLDatabase item result = _extract_item_attribute(mock_workspace, "$items.SQLDatabase.TestSQLDatabase.sqlendpoint", False) assert result == "test-sql-server.database.fabric.microsoft.com,1433" result = _extract_item_attribute(mock_workspace, "$items.SQLDatabase.TestSQLDatabase.$sqlendpoint", False) assert result == "test-sql-server.database.fabric.microsoft.com,1433" # Test with valid eventhouse item result = _extract_item_attribute(mock_workspace, "$items.Eventhouse.Test Eventhouse.queryserviceuri", False) assert result == "eventhouse-query-uri" result = _extract_item_attribute(mock_workspace, "$items.Eventhouse.Test Eventhouse.$queryserviceuri", False) assert result == "eventhouse-query-uri" def test_extract_item_attribute_invalid(self, mock_workspace): """Tests _extract_item_attribute with invalid variable cases.""" # Test with invalid syntax with pytest.raises(ParsingError, match="Invalid \\$items variable syntax"): _extract_item_attribute(mock_workspace, "$items.Notebook", False) with pytest.raises(ParsingError, match="Invalid \\$items variable syntax"): _extract_item_attribute(mock_workspace, "$items.Notebook.Test Notebook", False) # Test with too many segments - now check for invalid attribute instead of invalid syntax mock_items_attr_lookup = list(constants.ITEM_ATTR_LOOKUP) with pytest.raises( ParsingError, match=re.escape(f"Attribute 'extra' is invalid. Supported attributes: {mock_items_attr_lookup}"), ): _extract_item_attribute(mock_workspace, "$items.Notebook.Test Notebook.id.extra", False) def test_extract_item_attribute_get_dataflow_name(self, mock_workspace): """Test _extract_item_attribute with special handling for Dataflow references.""" # Test when Dataflow references another Dataflow in the repository result = _extract_item_attribute(mock_workspace, "$items.Dataflow.Source Dataflow.id", True) assert result == "Source Dataflow" # Test when source Dataflow doesn't exist in repository - should return None result = _extract_item_attribute(mock_workspace, "$items.Dataflow.NonExistentDataflow.id", True) assert result is None # Test when source Dataflow type doesn't match (case sensitive) - should return None result = _extract_item_attribute(mock_workspace, "$items.dataflow.Source Dataflow.id", get_dataflow_name=True) assert result is None # Test when source Dataflow name doesn't match (case sensitive) - should return None result = _extract_item_attribute(mock_workspace, "$items.Dataflow.source dataflow.id", get_dataflow_name=True) assert result is None def test_extract_workspace_id_direct(self, mock_workspace): """Tests _extract_workspace_id with direct workspace ID variable.""" from fabric_cicd._parameter._utils import _extract_workspace_id # Test with $workspace.id - should return workspace_id directly result = _extract_workspace_id(mock_workspace, "$workspace.id") assert result == "mock-workspace-id" result = _extract_workspace_id(mock_workspace, "$workspace.$id") assert result == "mock-workspace-id" def test_extract_workspace_id_resolve(self, mock_workspace): """Tests _extract_workspace_id with workspace name resolution.""" from fabric_cicd._parameter._utils import _extract_workspace_id # Mock the _resolve_workspace_id method mock_workspace._resolve_workspace_id.return_value = "resolved-workspace-id" # Test with both backward-compatible and explicit ID syntaxes result = _extract_workspace_id(mock_workspace, "$workspace.test_workspace") assert result == "resolved-workspace-id" mock_workspace._resolve_workspace_id.assert_called_once_with("test_workspace") mock_workspace._resolve_workspace_id.reset_mock() result = _extract_workspace_id(mock_workspace, "$workspace.test_workspace.$id") assert result == "resolved-workspace-id" mock_workspace._resolve_workspace_id.assert_called_once_with("test_workspace") def test_extract_workspace_id_with_workspace_name_variable(self, mock_workspace): """Tests _extract_workspace_id with workspace name variable.""" from fabric_cicd._parameter._utils import _extract_workspace_id mock_workspace._resolve_workspace_name = mock.MagicMock(return_value="My Target Workspace [PPE]") result = _extract_workspace_id(mock_workspace, "$workspace.$name") assert result == "My Target Workspace [PPE]" mock_workspace._resolve_workspace_name.assert_called_once_with() def test_extract_workspace_id_name_encoded(self, mock_workspace): """Tests _extract_workspace_id with $workspace.$name_encoded returns URL-encoded name.""" from fabric_cicd._parameter._utils import _extract_workspace_id mock_workspace._resolve_workspace_name = mock.MagicMock(return_value="My Target Workspace [PPE]") result = _extract_workspace_id(mock_workspace, "$workspace.$name_encoded") assert result == "My%20Target%20Workspace%20%5BPPE%5D" mock_workspace._resolve_workspace_name.assert_called_once_with() def test_extract_workspace_id_resolve_error(self, mock_workspace): """Tests _extract_workspace_id when workspace name resolution fails.""" from fabric_cicd._parameter._utils import _extract_workspace_id # Mock the _resolve_workspace_id method to raise InputError mock_workspace._resolve_workspace_id.side_effect = InputError("Workspace name not found", logger) # Should re-raise the same InputError with pytest.raises(InputError, match=r"Workspace name not found"): _extract_workspace_id(mock_workspace, "$workspace.unknown_workspace") def test_extract_workspace_id_general_error(self, mock_workspace): """Tests _extract_workspace_id with unexpected errors.""" from fabric_cicd._parameter._utils import _extract_workspace_id # Mock the _resolve_workspace_id method to raise a general exception mock_workspace._resolve_workspace_id.side_effect = Exception("Unexpected error") # Should wrap general exceptions in ParsingError with pytest.raises(ParsingError, match=r"Error parsing \$workspace variable"): _extract_workspace_id(mock_workspace, "$workspace.test_workspace") def test_extract_item_attribute_null_return(self, mock_workspace): """Tests _extract_item_attribute cases that return None.""" # Test with non-Dataflow item in get_dataflow_name mode should return None result = _extract_item_attribute(mock_workspace, "$items.Lakehouse.Test Lakehouse.id", True) assert result is None # Test with Dataflow item, but incorrect attribute should return None result = _extract_item_attribute(mock_workspace, "$items.Dataflow.Source Dataflow.sqlendpoint", True) assert result is None def test_extract_item_attribute_invalid_attribute(self, mock_workspace): """Tests _extract_item_attribute with invalid attribute.""" # Test with invalid attribute mock_items_attr_lookup = list(constants.ITEM_ATTR_LOOKUP) with pytest.raises( ParsingError, match=re.escape(f"Attribute 'guid' is invalid. Supported attributes: {mock_items_attr_lookup}"), ): _extract_item_attribute(mock_workspace, "$items.Dataflow.Source Dataflow.guid", True) def test_extract_workspace_id_with_item_lookup(self, mock_workspace): """Tests _extract_workspace_id with item lookup in another workspace.""" from fabric_cicd._parameter._utils import _extract_workspace_id # Mock the _resolve_workspace_id method mock_workspace._resolve_workspace_id.return_value = "resolved-workspace-id" # Mock the _lookup_item_attribute method mock_workspace._lookup_item_attribute = mock.MagicMock(return_value="item-123-id") # Test with $workspace..$items...$id format result = _extract_workspace_id(mock_workspace, "$workspace.test_workspace.$items.Notebook.Test Notebook.$id") assert result == "item-123-id" mock_workspace._resolve_workspace_id.assert_called_once_with("test_workspace") # attribute should be passed without leading '$' mock_workspace._lookup_item_attribute.assert_called_once_with( "resolved-workspace-id", "Notebook", "Test Notebook", "id" ) def test_extract_workspace_id_with_item_lookup_not_found(self, mock_workspace): """Tests _extract_workspace_id when item lookup fails.""" from fabric_cicd._parameter._utils import _extract_workspace_id # Mock the _resolve_workspace_id method mock_workspace._resolve_workspace_id.return_value = "resolved-workspace-id" # Mock the _lookup_item_attribute method to raise InputError (item not found) error_msg = ( "Failed to look up item in workspace: resolved-workspace-id, item_type: Notebook, item_name: Test Notebook" ) mock_workspace._lookup_item_attribute = mock.MagicMock(side_effect=InputError(error_msg, logger)) # Should re-raise the InputError with pytest.raises(InputError, match=re.escape(error_msg)): _extract_workspace_id(mock_workspace, "$workspace.test_workspace.$items.Notebook.Test Notebook.$id") mock_workspace._resolve_workspace_id.assert_called_once_with("test_workspace") mock_workspace._lookup_item_attribute.assert_called_once_with( "resolved-workspace-id", "Notebook", "Test Notebook", "id" ) @pytest.mark.parametrize( "invalid_var", [ "$workspace.$items.Notebook.Test Notebook.$id", # Missing workspace name "$workspace.test_workspace.$items.InvalidType.Test Notebook.$id", # Invalid item type "$workspace.test_workspace.$items.Notebook.$id", # Missing item name ], ) def test_extract_workspace_id_with_item_lookup_invalid_format(self, mock_workspace, invalid_var): """Tests _extract_workspace_id with invalid item lookup format.""" from fabric_cicd._parameter._utils import _extract_workspace_id # Test with invalid formats with pytest.raises(ParsingError): _extract_workspace_id(mock_workspace, invalid_var) def test_extract_workspace_id_with_item_lookup_sqlendpoint(self, mock_workspace): """Tests _extract_workspace_id resolves sqlendpoint from another workspace via $items reference.""" from fabric_cicd._parameter._utils import _extract_workspace_id mock_workspace._resolve_workspace_id.return_value = "resolved-workspace-id" mock_workspace._lookup_item_attribute = mock.MagicMock(return_value="lakehouse-endpoint-value") result = _extract_workspace_id( mock_workspace, "$workspace.test_workspace.$items.Lakehouse.Test_Lakehouse.$sqlendpoint" ) assert result == "lakehouse-endpoint-value" mock_workspace._resolve_workspace_id.assert_called_once_with("test_workspace") mock_workspace._lookup_item_attribute.assert_called_once_with( "resolved-workspace-id", "Lakehouse", "Test_Lakehouse", "sqlendpoint" ) def test_extract_workspace_id_with_item_lookup_queryserviceuri(self, mock_workspace): """Tests _extract_workspace_id resolves queryserviceuri from another workspace via $items reference.""" from fabric_cicd._parameter._utils import _extract_workspace_id mock_workspace._resolve_workspace_id.return_value = "resolved-workspace-id" mock_workspace._lookup_item_attribute = mock.MagicMock(return_value="eventhouse-query-uri-value") result = _extract_workspace_id( mock_workspace, "$workspace.test_workspace.$items.Eventhouse.Test Eventhouse.$queryserviceuri" ) assert result == "eventhouse-query-uri-value" mock_workspace._resolve_workspace_id.assert_called_once_with("test_workspace") mock_workspace._lookup_item_attribute.assert_called_once_with( "resolved-workspace-id", "Eventhouse", "Test Eventhouse", "queryserviceuri" ) def test_extract_workspace_id_with_item_lookup_sqlendpointid(self, mock_workspace): """Tests _extract_workspace_id resolves sqlendpointid from another workspace via $items reference.""" from fabric_cicd._parameter._utils import _extract_workspace_id mock_workspace._resolve_workspace_id.return_value = "resolved-workspace-id" mock_workspace._lookup_item_attribute = mock.MagicMock(return_value="lakehouse-sql-endpoint-id-value") result = _extract_workspace_id( mock_workspace, "$workspace.test_workspace.$items.Lakehouse.Test_Lakehouse.$sqlendpointid" ) assert result == "lakehouse-sql-endpoint-id-value" mock_workspace._resolve_workspace_id.assert_called_once_with("test_workspace") mock_workspace._lookup_item_attribute.assert_called_once_with( "resolved-workspace-id", "Lakehouse", "Test_Lakehouse", "sqlendpointid" ) def test_extract_replace_value_workspace_name(self, mock_workspace): """Tests extract_replace_value returns workspace name for $workspace.$name.""" mock_workspace._resolve_workspace_name = mock.MagicMock(return_value="My Target Workspace [PPE]") result = extract_replace_value(mock_workspace, "$workspace.$name") assert result == "My Target Workspace [PPE]" # $workspace.name (without $) should resolve "name" as a workspace name, not return display name mock_workspace._resolve_workspace_id.return_value = "resolved-id-for-name" result = extract_replace_value(mock_workspace, "$workspace.name") assert result == "resolved-id-for-name" mock_workspace._resolve_workspace_id.assert_called_once_with("name") mock_workspace._resolve_workspace_id.reset_mock() result = extract_replace_value(mock_workspace, "$workspace.TestWorkspace.$id") assert result == "resolved-id-for-name" mock_workspace._resolve_workspace_id.assert_called_once_with("TestWorkspace") def test_extract_parameter_filters(self, mock_workspace): """Tests extract_parameter_filters function.""" # Test with all filters param_dict = {"item_type": "Notebook", "item_name": "TestNotebook", "file_path": "path/to/file.txt"} with mock.patch("fabric_cicd._parameter._utils.process_input_path") as mock_process: # Return a list of Path objects as expected processed_path = Path("processed/path") mock_process.return_value = [processed_path] item_type, item_name, file_path = extract_parameter_filters(mock_workspace, param_dict) assert item_type == "Notebook" assert item_name == "TestNotebook" # Assert that file_path is a list containing the processed path assert file_path == [processed_path] mock_process.assert_called_once_with(mock_workspace.repository_directory, "path/to/file.txt") # Test with missing filters param_dict = {} with mock.patch("fabric_cicd._parameter._utils.process_input_path") as mock_process: # When no file_path in param_dict, process_input_path should return an empty list mock_process.return_value = [] item_type, item_name, file_path = extract_parameter_filters(mock_workspace, param_dict) assert item_type is None assert item_name is None assert file_path == [] def test_check_parameter_structure(self): """Tests _check_parameter_structure function.""" # Test with valid list assert _check_parameter_structure([1, 2, 3]) is True assert _check_parameter_structure([]) is True # Test with invalid types assert _check_parameter_structure("string") is False assert _check_parameter_structure(123) is False assert _check_parameter_structure({"key": "value"}) is False assert _check_parameter_structure(None) is False def test_is_valid_structure(self): """Tests is_valid_structure function.""" # Test with valid structures valid_dict = { "find_replace": [{"find_value": "test"}], "key_value_replace": [{"find_key": "$.test"}], "spark_pool": [{"instance_pool_id": "test"}], "semantic_model_binding": [{"connection_id": "test"}], } assert is_valid_structure(valid_dict) is True assert is_valid_structure(valid_dict, "find_replace") is True # Test with invalid structures invalid_dict = { "find_replace": "not a list", "key_value_replace": [{"find_key": "$.test"}], "semantic_model_binding": "not a list", } assert is_valid_structure(invalid_dict) is False assert is_valid_structure(invalid_dict, "find_replace") is False # Test with missing parameters missing_dict = { "unknown_param": [{"test": "value"}], } assert is_valid_structure(missing_dict) is False # Test with empty dict assert is_valid_structure({}) is False def test_is_valid_structure_semantic_model_binding_new_format(self): """Tests is_valid_structure with new dictionary format for semantic_model_binding.""" # New format with default and models valid_new_format = { "semantic_model_binding": { "default": { "connection_id": { "PPE": "76e05dfe-9855-4e3d-a410-1dda048dbe99", } }, "models": [ { "semantic_model_name": ["Model1", "Model2"], "connection_id": { "PPE": "f96870d5-5f86-49ad-bf41-5967fd7c1c6d", }, } ], } } assert is_valid_structure(valid_new_format) is True assert is_valid_structure(valid_new_format, "semantic_model_binding") is True # New format with only default valid_default_only = { "semantic_model_binding": {"default": {"connection_id": {"_ALL_": "76e05dfe-9855-4e3d-a410-1dda048dbe99"}}} } assert is_valid_structure(valid_default_only) is True # New format with only models valid_models_only = { "semantic_model_binding": { "models": [ { "semantic_model_name": "SingleModel", "connection_id": {"DEV": "76e05dfe-9855-4e3d-a410-1dda048dbe99"}, } ] } } assert is_valid_structure(valid_models_only) is True # String is invalid invalid_string = {"semantic_model_binding": "not valid"} assert is_valid_structure(invalid_string) is False # Empty dict is invalid (no bindings configured) invalid_empty = {"semantic_model_binding": {}} assert is_valid_structure(invalid_empty) is False @mock.patch("fabric_cicd._parameter._parameter.Parameter") @mock.patch("fabric_cicd._common._validate_input.validate_repository_directory") @mock.patch("fabric_cicd._common._validate_input.validate_item_type_in_scope") @mock.patch("fabric_cicd._common._validate_input.validate_environment") def test_validate_parameter_file(self, mock_validate_env, mock_validate_item_type, mock_validate_repo, mock_param): """Tests validate_parameter_file function with default parameters.""" # Setup mocks mock_validate_repo.return_value = Path("/mock/repo") mock_validate_item_type.return_value = ["Notebook", "Lakehouse"] mock_validate_env.return_value = "Test" mock_param_instance = mock.MagicMock() mock_param.return_value = mock_param_instance mock_param_instance._validate_parameter_file.return_value = True # Call the function from fabric_cicd._parameter._utils import validate_parameter_file # Patch the FabricEndpoint inside the test since we need it to run successfully with mock.patch("fabric_cicd._common._fabric_endpoint.FabricEndpoint", return_value=mock.MagicMock()): result = validate_parameter_file( repository_directory=Path("/mock/repo"), item_type_in_scope=["Notebook", "Lakehouse"], environment="Test", ) # Verify the result assert result is True mock_param.assert_called_once_with( repository_directory=Path("/mock/repo"), item_type_in_scope=["Notebook", "Lakehouse"], environment="Test", parameter_file_name="parameter.yml", parameter_file_path=None, ) mock_param_instance._validate_parameter_file.assert_called_once() @mock.patch("fabric_cicd._parameter._parameter.Parameter") @mock.patch("fabric_cicd._common._validate_input.validate_repository_directory") @mock.patch("fabric_cicd._common._validate_input.validate_item_type_in_scope") @mock.patch("fabric_cicd._common._validate_input.validate_environment") def test_validate_parameter_file_with_custom_file_name( self, mock_validate_env, mock_validate_item_type, mock_validate_repo, mock_param ): """Tests validate_parameter_file function with custom parameter file name.""" # Setup mocks mock_validate_repo.return_value = Path("/mock/repo") mock_validate_item_type.return_value = ["Notebook", "Lakehouse"] mock_validate_env.return_value = "Test" mock_param_instance = mock.MagicMock() mock_param.return_value = mock_param_instance mock_param_instance._validate_parameter_file.return_value = True # Call the function from fabric_cicd._parameter._utils import validate_parameter_file # Patch the FabricEndpoint inside the test since we need it to run successfully with mock.patch("fabric_cicd._common._fabric_endpoint.FabricEndpoint", return_value=mock.MagicMock()): result = validate_parameter_file( repository_directory=Path("/mock/repo"), item_type_in_scope=["Notebook", "Lakehouse"], environment="Test", parameter_file_name="custom_params.yml", ) # Verify the result assert result is True mock_param.assert_called_once_with( repository_directory=Path("/mock/repo"), item_type_in_scope=["Notebook", "Lakehouse"], environment="Test", parameter_file_name="custom_params.yml", parameter_file_path=None, ) mock_param_instance._validate_parameter_file.assert_called_once() @mock.patch("fabric_cicd._parameter._parameter.Parameter") @mock.patch("fabric_cicd._common._validate_input.validate_repository_directory") @mock.patch("fabric_cicd._common._validate_input.validate_item_type_in_scope") @mock.patch("fabric_cicd._common._validate_input.validate_environment") def test_validate_parameter_file_with_custom_file_path( self, mock_validate_env, mock_validate_item_type, mock_validate_repo, mock_param ): """Tests validate_parameter_file function with custom parameter file path.""" # Setup mocks mock_validate_repo.return_value = Path("/mock/repo") mock_validate_item_type.return_value = ["Notebook", "Lakehouse"] mock_validate_env.return_value = "Test" mock_param_instance = mock.MagicMock() mock_param.return_value = mock_param_instance mock_param_instance._validate_parameter_file.return_value = True # Call the function from fabric_cicd._parameter._utils import validate_parameter_file # Patch the FabricEndpoint inside the test since we need it to run successfully with mock.patch("fabric_cicd._common._fabric_endpoint.FabricEndpoint", return_value=mock.MagicMock()): result = validate_parameter_file( repository_directory=Path("/mock/repo"), item_type_in_scope=["Notebook", "Lakehouse"], environment="Test", parameter_file_path="/custom/path/to/parameters.yml", ) # Verify the result assert result is True mock_param.assert_called_once_with( repository_directory=Path("/mock/repo"), item_type_in_scope=["Notebook", "Lakehouse"], environment="Test", parameter_file_name="parameter.yml", parameter_file_path="/custom/path/to/parameters.yml", ) mock_param_instance._validate_parameter_file.assert_called_once() @mock.patch("fabric_cicd._parameter._parameter.Parameter") @mock.patch("fabric_cicd._common._validate_input.validate_repository_directory") @mock.patch("fabric_cicd._common._validate_input.validate_item_type_in_scope") @mock.patch("fabric_cicd._common._validate_input.validate_environment") def test_validate_parameter_file_with_both_custom_name_and_path( self, mock_validate_env, mock_validate_item_type, mock_validate_repo, mock_param ): """Tests validate_parameter_file function with both custom file name and path.""" # Setup mocks mock_validate_repo.return_value = Path("/mock/repo") mock_validate_item_type.return_value = ["Notebook", "Lakehouse"] mock_validate_env.return_value = "Test" mock_param_instance = mock.MagicMock() mock_param.return_value = mock_param_instance mock_param_instance._validate_parameter_file.return_value = True # Call the function from fabric_cicd._parameter._utils import validate_parameter_file # Patch the FabricEndpoint inside the test since we need it to run successfully with mock.patch("fabric_cicd._common._fabric_endpoint.FabricEndpoint", return_value=mock.MagicMock()): result = validate_parameter_file( repository_directory=Path("/mock/repo"), item_type_in_scope=["Notebook", "Lakehouse"], environment="Test", parameter_file_name="custom_params.yml", parameter_file_path="/custom/path/to/parameters.yml", ) # Verify the result assert result is True mock_param.assert_called_once_with( repository_directory=Path("/mock/repo"), item_type_in_scope=["Notebook", "Lakehouse"], environment="Test", parameter_file_name="custom_params.yml", parameter_file_path="/custom/path/to/parameters.yml", ) mock_param_instance._validate_parameter_file.assert_called_once() @mock.patch("fabric_cicd._parameter._parameter.Parameter") @mock.patch("fabric_cicd._common._validate_input.validate_repository_directory") @mock.patch("fabric_cicd._common._validate_input.validate_item_type_in_scope") @mock.patch("fabric_cicd._common._validate_input.validate_environment") def test_validate_parameter_file_with_none_item_type_in_scope( self, mock_validate_env, mock_validate_item_type, mock_validate_repo, mock_param ): """Tests validate_parameter_file function when item_type_in_scope is omitted (None).""" # Setup mocks mock_validate_repo.return_value = Path("/mock/repo") # Mock validate_item_type_in_scope to return all supported types when None is passed mock_validate_item_type.return_value = list(constants.ACCEPTED_ITEM_TYPES) mock_validate_env.return_value = "Test" mock_param_instance = mock.MagicMock() mock_param.return_value = mock_param_instance mock_param_instance._validate_parameter_file.return_value = True # Call the function from fabric_cicd._parameter._utils import validate_parameter_file # Patch the FabricEndpoint inside the test since we need it to run successfully with mock.patch("fabric_cicd._common._fabric_endpoint.FabricEndpoint", return_value=mock.MagicMock()): result = validate_parameter_file( repository_directory=Path("/mock/repo"), environment="Test", ) # Verify the result assert result is True # Verify that validate_item_type_in_scope was called with None mock_validate_item_type.assert_called_once_with(None) # Verify Parameter was called with all supported item types mock_param.assert_called_once_with( repository_directory=Path("/mock/repo"), item_type_in_scope=list(constants.ACCEPTED_ITEM_TYPES), environment="Test", parameter_file_name="parameter.yml", parameter_file_path=None, ) mock_param_instance._validate_parameter_file.assert_called_once() def test_find_match(self): """Tests _find_match function with various inputs.""" # Test with None param_value assert _find_match(None, "value") is True # Test with string param_value assert _find_match("value", "value") is True assert _find_match("value", "other") is False # Test with list param_value assert _find_match(["value1", "value2"], "value1") is True assert _find_match(["value1", "value2"], "value3") is False # Test with list of Paths path_list = [Path("test1.txt"), Path("test2.txt")] assert _find_match(path_list, Path("test1.txt")) is True assert _find_match(path_list, Path("test3.txt")) is False # Test with invalid type assert _find_match(123, "value") is False def test_check_replacement(self, temp_repository): """Tests check_replacement function with various combinations of inputs.""" file_path = temp_repository / "file1.txt" # Test with no filters assert check_replacement(None, None, None, "type1", "name1", file_path) is True # Test with matching filters assert check_replacement("type1", "name1", [file_path], "type1", "name1", file_path) is True # Test with non-matching filters assert check_replacement("type2", "name1", [file_path], "type1", "name1", file_path) is False assert check_replacement("type1", "name2", [file_path], "type1", "name1", file_path) is False assert check_replacement("type1", "name1", [Path("other.txt")], "type1", "name1", file_path) is False # Test with combination of matching/non-matching filters assert check_replacement("type1", "name2", [file_path], "type1", "name1", file_path) is False assert check_replacement("type1", "name1", [Path("other.txt")], "type1", "name1", file_path) is False def test_replace_key_value_valid_json(self, mock_workspace): """Tests replace_key_value with valid JSON content and environment.""" # Test JSON with server host configuration test_json = '{"server": {"host": "localhost", "port": 8080}}' param_dict = { "find_key": "$.server.host", "replace_value": {"dev": "dev-server.example.com", "prod": "prod-server.example.com"}, } # Test successful replacement for dev environment result = replace_key_value(mock_workspace, param_dict, test_json, "dev") result_data = json.loads(result) assert result_data["server"]["host"] == "dev-server.example.com" assert result_data["server"]["port"] == 8080 # Verify other values unchanged # Test successful replacement for prod environment result = replace_key_value(mock_workspace, param_dict, test_json, "prod") result_data = json.loads(result) assert result_data["server"]["host"] == "prod-server.example.com" def test_replace_key_value_environment_not_found(self, mock_workspace): """Tests replace_key_value when environment is not in the replace_value dictionary.""" test_json = '{"server": {"host": "localhost", "port": 8080}}' param_dict = { "find_key": "$.server.host", "replace_value": {"dev": "dev-server.example.com", "prod": "prod-server.example.com"}, } # Test when environment not in replace_value result = replace_key_value(mock_workspace, param_dict, test_json, "test") result_data = json.loads(result) assert result_data["server"]["host"] == "localhost" # Original value unchanged def test_replace_key_value_invalid_json(self, mock_workspace): """Tests replace_key_value with invalid JSON content.""" invalid_json = "{invalid json content}" param_dict = {"find_key": "$.server.host", "replace_value": {"dev": "test-server"}} # JSONDecodeError will be raised for invalid JSON and wrapped in ValueError with pytest.raises(ValueError, match="Expecting property name"): replace_key_value(mock_workspace, param_dict, invalid_json, "dev") def test_replace_key_value(self, mock_workspace): """Test replace_key_value function with JSON content.""" # Create test parameter dictionary and JSON content param_dict = { "find_key": "$.server.host", "replace_value": {"dev": "dev-server.example.com", "prod": "prod-server.example.com"}, } json_content = '{"server": {"host": "localhost", "port": 8080}}' # Test successful replacement result = replace_key_value(mock_workspace, param_dict, json_content, "dev") # Parse the JSON result and check the exact value (avoid substring sanitization issues) result_json = json.loads(result) assert result_json["server"]["host"] == "dev-server.example.com" # Test with environment not in replace_value result = replace_key_value(mock_workspace, param_dict, json_content, "test") result_json = json.loads(result) assert result_json["server"]["host"] == "localhost" # Test with invalid JSON content with pytest.raises(ValueError, match="Expecting property name"): replace_key_value(mock_workspace, param_dict, "{invalid json}", "dev") def test_replace_key_value_with_items_notation(self, mock_workspace): """Test replace_key_value function with $items notation.""" # Mock the workspace to return item attributes mock_workspace.workspace_items = { "Lakehouse": { "TestLakehouse": { "id": "test-lakehouse-id-12345", "sqlendpoint": "test-lakehouse.database.windows.net", } }, "Warehouse": { "TestWarehouse": { "id": "test-warehouse-id-67890", "sqlendpoint": "test-warehouse.database.windows.net", } }, } mock_workspace._refresh_deployed_items = MagicMock() # Test JSON with item references test_json = '{"lakehouse": {"id": "placeholder-id", "endpoint": "placeholder-endpoint"}}' # Test with $items notation for lakehouse id param_dict = { "find_key": "$.lakehouse.id", "replace_value": { "dev": "$items.Lakehouse.TestLakehouse.$id", "prod": "$items.Lakehouse.TestLakehouse.$id", }, } # Test successful replacement with $items notation result = replace_key_value(mock_workspace, param_dict, test_json, "dev") result_data = json.loads(result) assert result_data["lakehouse"]["id"] == "test-lakehouse-id-12345" assert result_data["lakehouse"]["endpoint"] == "placeholder-endpoint" # Unchanged # Test with $items notation for sqlendpoint param_dict = { "find_key": "$.lakehouse.endpoint", "replace_value": { "dev": "$items.Lakehouse.TestLakehouse.$sqlendpoint", "prod": "$items.Warehouse.TestWarehouse.$sqlendpoint", }, } result = replace_key_value(mock_workspace, param_dict, test_json, "dev") result_data = json.loads(result) assert result_data["lakehouse"]["endpoint"] == "test-lakehouse.database.windows.net" result = replace_key_value(mock_workspace, param_dict, test_json, "prod") result_data = json.loads(result) assert result_data["lakehouse"]["endpoint"] == "test-warehouse.database.windows.net" def test_replace_key_value_with_items_notation_and_non_string_values(self, mock_workspace): """Test replace_key_value function with $items notation mixed with other value types.""" # Mock the workspace to return item attributes mock_workspace.workspace_items = { "Lakehouse": { "TestLakehouse": { "id": "test-lakehouse-id-12345", } }, } mock_workspace._refresh_deployed_items = MagicMock() # Test JSON with mixed value types test_json = '{"config": {"enabled": false, "count": 100, "lakehouse_id": "placeholder"}}' # Test with boolean value (should not process as $items) param_dict = { "find_key": "$.config.enabled", "replace_value": {"dev": True, "prod": False}, } result = replace_key_value(mock_workspace, param_dict, test_json, "dev") result_data = json.loads(result) assert result_data["config"]["enabled"] is True # Test with integer value (should not process as $items) param_dict = { "find_key": "$.config.count", "replace_value": {"dev": 200, "prod": 300}, } result = replace_key_value(mock_workspace, param_dict, test_json, "dev") result_data = json.loads(result) assert result_data["config"]["count"] == 200 # Test with string $items notation param_dict = { "find_key": "$.config.lakehouse_id", "replace_value": { "dev": "$items.Lakehouse.TestLakehouse.$id", "prod": "$items.Lakehouse.TestLakehouse.$id", }, } result = replace_key_value(mock_workspace, param_dict, test_json, "dev") result_data = json.loads(result) assert result_data["config"]["lakehouse_id"] == "test-lakehouse-id-12345" def test_replace_key_value_yaml_valid(self, mock_workspace): """Tests replace_key_value_yaml with valid YAML content and environment.""" # Test YAML with server host configuration test_yaml = """server: host: localhost port: 8080 """ param_dict = { "find_key": "$.server.host", "replace_value": {"dev": "dev-server.example.com", "prod": "prod-server.example.com"}, } # Test successful replacement for dev environment result = replace_key_value(mock_workspace, param_dict, test_yaml, "dev", is_yaml=True) result_data = yaml.safe_load(result) assert result_data["server"]["host"] == "dev-server.example.com" assert result_data["server"]["port"] == 8080 # Verify other values unchanged # Test successful replacement for prod environment result = replace_key_value(mock_workspace, param_dict, test_yaml, "prod", is_yaml=True) result_data = yaml.safe_load(result) assert result_data["server"]["host"] == "prod-server.example.com" def test_replace_key_value_yaml_environment_not_found(self, mock_workspace): """Tests replace_key_value_yaml when environment is not in the replace_value dictionary.""" test_yaml = """server: host: localhost port: 8080 """ param_dict = { "find_key": "$.server.host", "replace_value": {"dev": "dev-server.example.com", "prod": "prod-server.example.com"}, } # Test when environment not in replace_value result = replace_key_value(mock_workspace, param_dict, test_yaml, "test", is_yaml=True) result_data = yaml.safe_load(result) assert result_data["server"]["host"] == "localhost" # Original value unchanged def test_replace_key_value_yaml_invalid(self, mock_workspace): """Tests replace_key_value_yaml with invalid YAML content.""" invalid_yaml = "invalid: yaml: content: [unclosed" param_dict = {"find_key": "$.server.host", "replace_value": {"dev": "test-server"}} # YAMLError will be raised for invalid YAML and wrapped in ValueError with pytest.raises(ValueError, match="mapping values are not allowed"): replace_key_value(mock_workspace, param_dict, invalid_yaml, "dev", is_yaml=True) def test_replace_key_value_yaml_empty_content(self, mock_workspace): """Tests replace_key_value_yaml with empty YAML content.""" empty_yaml = "" param_dict = {"find_key": "$.server.host", "replace_value": {"dev": "test-server"}} # Empty YAML should return as-is result = replace_key_value(mock_workspace, param_dict, empty_yaml, "dev", is_yaml=True) assert result == empty_yaml def test_replace_key_value_yaml_nested_structure(self, mock_workspace): """Tests replace_key_value_yaml with nested YAML structure like SparkCompute.yml.""" # Test YAML similar to SparkCompute.yml test_yaml = """enable_native_execution_engine: false driver_cores: 8 driver_memory: 56g executor_cores: 8 executor_memory: 56g dynamic_executor_allocation: enabled: true min_executors: 1 max_executors: 9 runtime_version: "1.2" """ # Test replacing a nested value param_dict = { "find_key": "$.dynamic_executor_allocation.max_executors", "replace_value": {"dev": 5, "prod": 20}, } result = replace_key_value(mock_workspace, param_dict, test_yaml, "dev", is_yaml=True) result_data = yaml.safe_load(result) assert result_data["dynamic_executor_allocation"]["max_executors"] == 5 assert result_data["dynamic_executor_allocation"]["min_executors"] == 1 # Unchanged # Test replacing a top-level value param_dict = { "find_key": "$.driver_cores", "replace_value": {"dev": 4, "prod": 16}, } result = replace_key_value(mock_workspace, param_dict, test_yaml, "prod", is_yaml=True) result_data = yaml.safe_load(result) assert result_data["driver_cores"] == 16 def test_replace_key_value_yaml_with_items_notation(self, mock_workspace): """Test replace_key_value_yaml function with $items notation.""" # Mock the workspace to return item attributes mock_workspace.workspace_items = { "Lakehouse": { "TestLakehouse": { "id": "test-lakehouse-id-12345", "sqlendpoint": "test-lakehouse.database.windows.net", } }, } mock_workspace._refresh_deployed_items = MagicMock() # Test YAML with item references test_yaml = """lakehouse: id: placeholder-id endpoint: placeholder-endpoint """ # Test with $items notation for lakehouse id param_dict = { "find_key": "$.lakehouse.id", "replace_value": { "dev": "$items.Lakehouse.TestLakehouse.$id", "prod": "$items.Lakehouse.TestLakehouse.$id", }, } # Test successful replacement with $items notation result = replace_key_value(mock_workspace, param_dict, test_yaml, "dev", is_yaml=True) result_data = yaml.safe_load(result) assert result_data["lakehouse"]["id"] == "test-lakehouse-id-12345" assert result_data["lakehouse"]["endpoint"] == "placeholder-endpoint" # Unchanged def test_replace_key_value_yaml_with_non_string_values(self, mock_workspace): """Test replace_key_value_yaml function with non-string value types.""" # Test YAML with mixed value types test_yaml = """config: enabled: false count: 100 threshold: 0.5 items: - item1 - item2 """ # Test with boolean value param_dict = { "find_key": "$.config.enabled", "replace_value": {"dev": True, "prod": False}, } result = replace_key_value(mock_workspace, param_dict, test_yaml, "dev", is_yaml=True) result_data = yaml.safe_load(result) assert result_data["config"]["enabled"] is True # Test with integer value param_dict = { "find_key": "$.config.count", "replace_value": {"dev": 200, "prod": 300}, } result = replace_key_value(mock_workspace, param_dict, test_yaml, "dev", is_yaml=True) result_data = yaml.safe_load(result) assert result_data["config"]["count"] == 200 # Test with float value param_dict = { "find_key": "$.config.threshold", "replace_value": {"dev": 0.8, "prod": 0.95}, } result = replace_key_value(mock_workspace, param_dict, test_yaml, "dev", is_yaml=True) result_data = yaml.safe_load(result) assert result_data["config"]["threshold"] == 0.8 def test_replace_variables_in_parameter_file(self, monkeypatch): """Test replace_variables_in_parameter_file with feature flag enabled.""" # Set up test environment variables test_env_vars = { "$ENV:TEST_VAR": "test_value", "$ENV:ANOTHER_VAR": "another_value", "NORMAL_VAR": "normal_value", # Should be ignored } # Mock os.environ monkeypatch.setattr("os.environ", test_env_vars) # Mock feature flag to be enabled monkeypatch.setattr(constants, "FEATURE_FLAG", ["enable_environment_variable_replacement"]) # Test parameter file content with environment variables test_content = """ parameter: value: $ENV:TEST_VAR other: $ENV:ANOTHER_VAR normal: NORMAL_VAR """ result = replace_variables_in_parameter_file(test_content) # Verify replacements assert "value: test_value" in result assert "other: another_value" in result assert "normal: NORMAL_VAR" in result # Normal var unchanged def test_replace_variables_in_parameter_file_feature_disabled(self, monkeypatch): """Test replace_variables_in_parameter_file with feature flag disabled.""" # Set up test environment variables with $ENV: prefix test_env_vars = { "$ENV:TEST_VAR": "test_value", "$ENV:ANOTHER_VAR": "another_value", } # Mock os.environ monkeypatch.setattr("os.environ", test_env_vars) # Mock feature flag to be disabled (empty list) monkeypatch.setattr(constants, "FEATURE_FLAG", []) # Test parameter file content with environment variables test_content = """ parameter: value: $ENV:TEST_VAR other: $ENV:ANOTHER_VAR normal: NORMAL_VAR """ result = replace_variables_in_parameter_file(test_content) # Verify NO replacements occurred since feature is disabled # Environment variables should remain as-is in the output assert "$ENV:TEST_VAR" in result assert "$ENV:ANOTHER_VAR" in result assert "NORMAL_VAR" in result # Normal var unchanged # Make sure no replacements happened assert "test_value" not in result assert "another_value" not in result def test_replace_env_variables_in_content(self, monkeypatch): """Test replace_variables_in_parameter_file with feature flag enabled.""" # Set up test environment variables with $ENV: prefix # This is required because the function filters os.environ for keys starting with $ENV: test_env_vars = { "$ENV:TEST_VAR": "test_value", "$ENV:ANOTHER_VAR": "another_value", "NORMAL_VAR": "normal_value", # Should be ignored (no $ENV: prefix) } # Mock os.environ monkeypatch.setattr("os.environ", test_env_vars) # Mock feature flag to be enabled monkeypatch.setattr(constants, "FEATURE_FLAG", ["enable_environment_variable_replacement"]) # Test parameter file content with environment variables test_content = """ parameter: value: $ENV:TEST_VAR other: $ENV:ANOTHER_VAR normal: NORMAL_VAR """ result = replace_variables_in_parameter_file(test_content) # Verify replacements assert "value: test_value" in result assert "other: another_value" in result assert "normal: NORMAL_VAR" in result # Normal var unchanged def test_process_environment_key(self, mock_workspace): """Test process_environment_key function with ALL environment key replacement.""" # Test with ALL key only - should replace with target environment replace_value_dict_1 = {"_ALL_": "universal-value"} replace_value_dict_2 = {"_all_": "universal-value"} replace_value_dict_3 = {"_All_": "universal-value"} replace_value_dict_4 = {"ALL": "universal-value"} # Mock the workspace environment mock_workspace.environment = "TEST" # Call the function result_1 = process_environment_key(mock_workspace.environment, replace_value_dict_1) result_2 = process_environment_key(mock_workspace.environment, replace_value_dict_2) result_3 = process_environment_key(mock_workspace.environment, replace_value_dict_3) result_4 = process_environment_key(mock_workspace.environment, replace_value_dict_4) # Verify _ALL_ key is replaced with the target environment assert "_ALL_" not in result_1 assert "TEST" in result_1 assert result_1["TEST"] == "universal-value" # Verify _all_ key is replaced with the target environment assert "_all_" not in result_2 assert "TEST" in result_2 assert result_2["TEST"] == "universal-value" # Verify _All_ key is replaced with the target environment assert "_All_" not in result_3 assert "TEST" in result_3 assert result_3["TEST"] == "universal-value" # Verify ALL key is replaced with the target environment assert "ALL" in result_4 assert "TEST" not in result_4 assert result_4["ALL"] == "universal-value" assert result_1 == {"TEST": "universal-value"} assert result_1 == result_2 == result_3 != result_4 # Test without ALL key - should return unchanged dictionary replace_value_dict_5 = { "DEV": "dev-value", "PROD": "prod-value", } # Mock the workspace environment mock_workspace.environment = "TEST" # Call the function result = process_environment_key(mock_workspace.environment, replace_value_dict_5) # Dictionary should remain unchanged assert result == replace_value_dict_5 assert "TEST" not in result def test_validate_item_type_in_scope_with_none(self): """Tests validate_item_type_in_scope function when None is passed.""" from fabric_cicd._common._validate_input import validate_item_type_in_scope # Test with None - should return all accepted item types result = validate_item_type_in_scope(None) assert result == list(constants.ACCEPTED_ITEM_TYPES) assert len(result) > 0 # Ensure we got some types back # Verify a few expected types are in the result assert "Notebook" in result assert "DataPipeline" in result assert "Environment" in result def test_validate_item_type_in_scope_with_valid_list(self): """Tests validate_item_type_in_scope function with a valid list.""" from fabric_cicd._common._validate_input import validate_item_type_in_scope # Test with valid list valid_types = ["Notebook", "Lakehouse", "Environment"] result = validate_item_type_in_scope(valid_types) assert result == valid_types def test_validate_item_type_in_scope_with_invalid_type(self): """Tests validate_item_type_in_scope function with invalid item type.""" from fabric_cicd._common._validate_input import validate_item_type_in_scope # Test with invalid item type invalid_types = ["Notebook", "InvalidType", "Environment"] with pytest.raises(InputError, match="Invalid or unsupported item type: 'InvalidType'"): validate_item_type_in_scope(invalid_types) class TestPathUtilities: """Tests for path utility functions in _utils.py.""" def test_process_input_path_none(self, temp_repository): """Tests process_input_path with none input.""" result = process_input_path(temp_repository, None) assert result is None def test_process_input_path_string(self, temp_repository, monkeypatch): """Tests process_input_path with string input.""" # Mock the helper functions and glob.has_magic def mock_process_regular_path(path, repo, valid_paths, _): if path == "file1.txt": valid_paths.add(repo / "file1.txt") def mock_process_wildcard_path(path, repo, valid_paths, _): if path == "*.txt": valid_paths.add(repo / "file1.txt") valid_paths.add(repo / "file2.txt") def mock_has_magic(path): return "*" in path # Apply the mocks monkeypatch.setattr("fabric_cicd._parameter._utils._process_regular_path", mock_process_regular_path) monkeypatch.setattr("fabric_cicd._parameter._utils._process_wildcard_path", mock_process_wildcard_path) monkeypatch.setattr("glob.has_magic", mock_has_magic) # Test with string path result = process_input_path(temp_repository, "file1.txt") assert isinstance(result, list) assert len(result) == 1 assert result[0].name == "file1.txt" # Test with wildcard string result = process_input_path(temp_repository, "*.txt") assert isinstance(result, list) assert len(result) == 2 # Should find the 2 .txt files in root def test_process_input_path_list(self, temp_repository, monkeypatch): """Tests process_input_path with list input.""" # Create a mapping of paths to the files they should find path_results = { "file1.txt": [temp_repository / "file1.txt"], "*.json": [temp_repository / "file2.json"], "folder1/*.py": [temp_repository / "folder1" / "file3.py"], } # Mock the helper functions and glob.has_magic def mock_process_regular_path(path, _, valid_paths, __): if path in path_results and "*" not in path: valid_paths.update(path_results[path]) def mock_process_wildcard_path(path, _, valid_paths, __): if path in path_results and "*" in path: valid_paths.update(path_results[path]) def mock_has_magic(path): return "*" in path # Apply the mocks monkeypatch.setattr("fabric_cicd._parameter._utils._process_regular_path", mock_process_regular_path) monkeypatch.setattr("fabric_cicd._parameter._utils._process_wildcard_path", mock_process_wildcard_path) monkeypatch.setattr("glob.has_magic", mock_has_magic) # Test with list of paths including both regular and wildcard patterns paths = ["file1.txt", "*.json", "folder1/*.py"] result = process_input_path(temp_repository, paths) assert isinstance(result, list) assert len(result) == 3 # Should find file1.txt, file2.json, and folder1/file3.py assert any(p.name == "file1.txt" for p in result) assert any(p.name == "file2.json" for p in result) assert any(p.name == "file3.py" for p in result) def test_process_input_path_has_magic_exception(self, temp_repository, monkeypatch): """Tests process_input_path when glob.has_magic raises an exception.""" # Create a mock logger to verify logging mock_logger = mock.MagicMock() monkeypatch.setattr("fabric_cicd._parameter._utils.logger", mock_logger) # Mock glob.has_magic to raise an exception def mock_has_magic(_): msg = "Mock exception in has_magic" raise ValueError(msg) monkeypatch.setattr("glob.has_magic", mock_has_magic) # Test with a single path - should handle the exception gracefully result = process_input_path(temp_repository, "file1.txt", False) # Verify the result is a list and it's empty (since we couldn't process the path) assert isinstance(result, list) assert len(result) == 0 # Verify that the error was logged assert mock_logger.debug.called assert "Error checking for wildcard" in mock_logger.debug.call_args_list[0][0][0] mock_logger.reset_mock() # Test with a list of paths - should attempt to process each path but return empty list # since all paths will fail the glob.has_magic check with an exception result = process_input_path(temp_repository, ["file1.txt", "file2.txt"], False) assert isinstance(result, list) assert len(result) == 0 # Verify that errors were logged for both paths assert mock_logger.debug.call_count == 2 assert "Error checking for wildcard" in mock_logger.debug.call_args_list[0][0][0] assert "Error checking for wildcard" in mock_logger.debug.call_args_list[1][0][0] def test_resolve_input_path_with_invalid_wildcard_syntax(self, temp_repository, monkeypatch): """Tests _resolve_input_path when _validate_wildcard_syntax returns False.""" # Create a valid path in the temp repository valid_path = temp_repository / "test.txt" valid_path.write_text("test content") # Mock _validate_wildcard_syntax to return False for our test pattern def mock_validate_wildcard_syntax(pattern, _): return pattern != "invalid*.txt" # Return False only for our test pattern monkeypatch.setattr("fabric_cicd._parameter._utils._validate_wildcard_syntax", mock_validate_wildcard_syntax) # Use a public function that calls _resolve_input_path with wildcard=True result = process_input_path(temp_repository, "invalid*.txt") # Should be empty because the wildcard validation failed assert len(result) == 0 def test_process_input_path_some_invalid(self, temp_repository, monkeypatch): """Tests process_input_path with some invalid paths.""" # Create a mock logger mock_logger = mock.MagicMock() monkeypatch.setattr("fabric_cicd._parameter._utils.logger", mock_logger) # Create test files we need for this test (temp_repository / "valid_file.txt").write_text("valid content") # Mock glob.has_magic to succeed for specific paths and fail for others import glob as glob_module original_has_magic = glob_module.has_magic def mock_has_magic(path): if path == "error_path.txt": msg = "Mock error for specific path" raise ValueError(msg) return original_has_magic(path) monkeypatch.setattr("glob.has_magic", mock_has_magic) # Mock _resolve_file_path to return a valid path for specific files def mock_resolve_file_path(path, *_): if "valid_file.txt" in str(path): return path return None monkeypatch.setattr("fabric_cicd._parameter._utils._resolve_file_path", mock_resolve_file_path) # Test with a mix of valid and problematic paths result = process_input_path(temp_repository, ["valid_file.txt", "error_path.txt", "nonexistent_file.txt"]) # Should return only valid paths assert isinstance(result, list) assert len(result) == 1 assert "valid_file.txt" in str(result[0]) # Verify errors were logged for problematic paths assert mock_logger.debug.called assert any("Error checking for wildcard" in call[0][0] for call in mock_logger.debug.call_args_list) def test_process_wildcard_path(self, temp_repository, monkeypatch): """Tests _process_wildcard_path function.""" # Create the test files we need for this test (temp_repository / "file1.txt").write_text("content1") (temp_repository / "file2.txt").write_text("content2") (temp_repository / "folder2" / "file5.txt").write_text("content5") # We need to patch the actual Path.glob method with our own implementation original_glob = Path.glob def patched_glob(self, pattern): # Special case for our test - return predefined results if str(self) == str(temp_repository): if pattern == "*.txt": return [temp_repository / "file1.txt", temp_repository / "file2.txt"] if pattern == "**/*.txt": return [ temp_repository / "file1.txt", temp_repository / "file2.txt", temp_repository / "folder2" / "file5.txt", ] # Fall back to original method for other cases return original_glob(self, pattern) # Apply the patch monkeypatch.setattr(Path, "glob", patched_glob) # Set up a valid paths set valid_paths = set() mock_log = mock.MagicMock() # Mock _set_wildcard_path_pattern to return our test pattern def mock_set_pattern(pattern, _repo, _log): return "*.txt" if pattern == "*.txt" else "**/*.txt" monkeypatch.setattr("fabric_cicd._parameter._utils._set_wildcard_path_pattern", mock_set_pattern) # Mock _resolve_file_path to return valid paths def mock_resolve_path(path, _repo, _path_type, _log): return path monkeypatch.setattr("fabric_cicd._parameter._utils._resolve_file_path", mock_resolve_path) # Test with wildcard pattern for txt files _process_wildcard_path("*.txt", temp_repository, valid_paths, mock_log) assert len(valid_paths) == 2 # Should find file1.txt and file2.txt in root assert all(path.suffix == ".txt" for path in valid_paths) # Reset paths and test with recursive pattern valid_paths.clear() _process_wildcard_path("**/*.txt", temp_repository, valid_paths, mock_log) assert len(valid_paths) == 3 # Should find all .txt files (including in subdirectories) def test_process_regular_path(self, temp_repository, monkeypatch): """Tests _process_regular_path with regular file paths.""" # Set up a valid paths set valid_paths = set() mock_log = mock.MagicMock() # Mock _resolve_file_path to return valid paths for specific files def mock_resolve_file_path(path, _repo, _path_type, _log): if path.name == "file1.txt" or path.name == "file2.json": return path.resolve() return None monkeypatch.setattr("fabric_cicd._parameter._utils._resolve_file_path", mock_resolve_file_path) # Test with specific file path _process_regular_path("file1.txt", temp_repository, valid_paths, mock_log) assert len(valid_paths) == 1 assert next(iter(valid_paths)).name == "file1.txt" # Reset and test with absolute path valid_paths.clear() abs_path = str(temp_repository / "file2.json") _process_regular_path(abs_path, temp_repository, valid_paths, mock_log) assert len(valid_paths) == 1 assert next(iter(valid_paths)).name == "file2.json" # Test with nonexistent file valid_paths.clear() _process_regular_path("nonexistent.txt", temp_repository, valid_paths, mock_log) assert len(valid_paths) == 0 # Should not add nonexistent files def test_resolve_nonexistent_file_path(self, temp_repository): """Tests _resolve_file_path with nonexistent files.""" # Test nonexistent file file_path = temp_repository / "nonexistent.txt" result = _resolve_file_path(file_path, temp_repository, "Relative", logger.debug) assert result is None def test_resolve_directory_file_path(self, temp_repository): """Tests _resolve_file_path with directories.""" # Test with directory instead of file dir_path = temp_repository / "folder1" result = _resolve_file_path(dir_path, temp_repository, "Relative", logger.debug) assert result is None def test_resolve_input_path_absolute_path(self): """Test _resolve_input_path with absolute path.""" # Using a standard logger function format that takes a string message mock_logger = MagicMock() repo_dir = Path("c:/test_repo").resolve() # Make sure it's resolved # Test with absolute path outside repository outside_path = Path("c:/outside/file.txt").resolve() # Make sure it's resolved # Simulate a path outside the repo by mocking the relative_to method with mock.patch.object(Path, "relative_to", side_effect=ValueError("Path outside repo")): result = _resolve_file_path(outside_path, repo_dir, "Absolute", mock_logger) # Check that the function returns None (path rejected) assert result is None # Check that the logger was called with an error about the path being outside mock_logger.assert_called_once_with(f"Absolute path '{outside_path}' is outside the repository directory") def test_resolve_outside_repo_file_path(self, temp_repository): """Tests _resolve_file_path with paths outside the repository.""" # Create a file outside the repository outside_dir = Path(tempfile.mkdtemp()) try: outside_file = outside_dir / "outside.txt" outside_file.write_text("outside content") # Test with file outside repository result = _resolve_file_path(outside_file, temp_repository, "Absolute", logger.debug) assert result is None finally: shutil.rmtree(outside_dir) def test_resolve_invalid_file_path(self, temp_repository, monkeypatch): """Tests _resolve_file_path with a path that causes exception.""" # Set up a mock that raises an exception when checking if file exists def mock_path_exists(_): msg = "Permission denied" raise PermissionError(msg) # Apply the mock monkeypatch.setattr(Path, "exists", mock_path_exists) # Test the exception handling file_path = temp_repository / "file1.txt" result = _resolve_file_path(file_path, temp_repository, "Test", logger.debug) assert result is None def test_validate_wildcard_syntax_invalid(self): """Test _validate_wildcard_syntax with invalid wildcard syntax.""" # Create a mock function to pass as log_func mock_log_func = MagicMock() # Test with invalid recursive wildcard format - double asterisk without proper format # This will trigger the check: "**" in p and not ("**/" in p or "/**" in p) invalid_path = "src**invalid.py" # Missing slash between src and ** # Call the function being tested result = _validate_wildcard_syntax(invalid_path, mock_log_func) # Verify validation fails assert result is False # Check that log_func was called exactly once with the expected message mock_log_func.assert_called_once_with(f"Invalid recursive wildcard format (use **/ or /**): '{invalid_path}'") def test_valid_wildcard_syntax(self): """Tests that valid wildcard patterns pass validation.""" # Create a mock logger mock_log_func = mock.MagicMock() valid_patterns = [ "*.txt", "**/*.py", "folder1/*.json", "folder?/*.txt", "folder[1-3]/*.txt", "file[!1-3].txt", "file{1,2,3}.txt", "**/subfolder/*.md", ] for pattern in valid_patterns: assert _validate_wildcard_syntax(pattern, mock_log_func) is True, f"Pattern should be valid: {pattern}" mock_log_func.assert_not_called() # No errors should be logged def test_invalid_wildcard_syntax(self): """Tests that invalid wildcard patterns fail validation, including complex bracket/brace nesting issues.""" # Create a mock logger mock_log_func = mock.MagicMock() # Group 1: Basic validation errors basic_invalid_patterns = [ "", # Empty string " ", # Whitespace only "../file.txt", # Path traversal "folder/../file.txt", # Path traversal "..%2Ffile.txt", # Encoded path traversal ] # Group 2: Wildcard pattern errors wildcard_invalid_patterns = [ "/**/*/", # Invalid combination "**/**", # Invalid combination "folder//file.txt", # Double slashes "folder\\\\file.txt", # Double backslashes "**file.txt", # Incorrect recursive format "//**/test.txt", # Absolute path with recursive pattern ] # Group 3: Bracket/brace validation errors bracket_brace_invalid_patterns = [ "folder[].txt", # Empty brackets "folder[abc.txt", # Unclosed bracket "folder{}.txt", # Empty braces "folder{abc.txt", # Unclosed brace "folder{,}.txt", # Invalid comma in braces "folder{a,,b}.txt", # Empty option in braces "folder{abc}.txt", # Brace without comma "folder[a-", # Unclosed bracket with range ] # Test all invalid patterns all_invalid_patterns = basic_invalid_patterns + wildcard_invalid_patterns + bracket_brace_invalid_patterns for pattern in all_invalid_patterns: assert _validate_wildcard_syntax(pattern, mock_log_func) is False, f"Pattern should be invalid: {pattern}" mock_log_func.assert_called() # Error should be logged mock_log_func.reset_mock() complex_invalid_nested_patterns = [ "folder[[a[b]c].txt", # Unbalanced nested brackets "folder{a{b,c}.txt", # Unbalanced nested braces ] # Test more complex bracket/brace nesting scenarios for pattern in complex_invalid_nested_patterns: assert _validate_wildcard_syntax(pattern, mock_log_func) is False, f"Pattern should be invalid: {pattern}" mock_log_func.assert_called() # Error should be logged mock_log_func.reset_mock() def test_validate_nested_brackets_braces(self): """Tests the _validate_nested_brackets_braces function to ensure proper validation of bracket/brace nesting.""" from fabric_cicd._parameter._utils import _validate_nested_brackets_braces as validate_func mock_log_func = mock.MagicMock() valid_nested_patterns = [ "file[abc].txt", # Simple bracket "file{a,b,c}.txt", # Simple brace "file[abc]{1,2,3}.txt", # Both brackets and braces "file[a[b]c].txt", # Nested brackets (valid in some glob implementations) "file{a{b,c},d}.txt", # Nested braces "file[[]].txt", # Escaped bracket in character class "file[a-z].{txt,md}", # Multiple bracket/brace pairs ] # Test valid patterns for pattern in valid_nested_patterns: assert validate_func(pattern, mock_log_func) is True, f"Pattern should be valid: {pattern}" mock_log_func.assert_not_called() mock_log_func.reset_mock() invalid_nested_patterns = [ "file[abc.txt", # Unclosed bracket "file{a,b.txt", # Unclosed brace "file]abc[.txt", # Closing before opening "file}abc{.txt", # Closing before opening "file[abc}.txt", # Mismatched pairs "file{abc].txt", # Mismatched pairs "file[a{b]c}.txt", # Interleaved mismatched pairs "file{a[b}c].txt", # Interleaved mismatched pairs ] # Test invalid patterns for pattern in invalid_nested_patterns: assert validate_func(pattern, mock_log_func) is False, f"Pattern should be invalid: {pattern}" mock_log_func.assert_called_once() mock_log_func.reset_mock() ================================================ FILE: tests/test_publish.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Test publishing functionality including selective publishing based on repository content.""" import json import logging import tempfile from pathlib import Path from typing import Optional from unittest.mock import MagicMock, patch import pytest from fixtures.credentials import DummyTokenCredential import fabric_cicd.publish as publish from fabric_cicd import constants from fabric_cicd._common._exceptions import InputError from fabric_cicd._items._notebook import NotebookPublisher from fabric_cicd.constants import API_FORMAT_MAPPING, ItemType from fabric_cicd.fabric_workspace import FabricWorkspace # ============================================================================= # Shared Fixtures and Helpers # ============================================================================= @pytest.fixture def mock_endpoint(): """Mock FabricEndpoint to avoid real API calls.""" mock = MagicMock() def mock_invoke(method, url, **_kwargs): if method == "GET" and "workspaces" in url and not url.endswith("/items"): return {"body": {"value": [], "capacityId": "test-capacity"}} if method == "GET" and url.endswith("/items"): return {"body": {"value": []}} if method == "POST" and url.endswith("/folders"): return {"body": {"id": "mock-folder-id"}} if method == "POST" and url.endswith("/items"): return {"body": {"id": "mock-item-id", "workspaceId": "mock-workspace-id"}} return {"body": {"value": [], "capacityId": "test-capacity"}} mock.invoke.side_effect = mock_invoke return mock @pytest.fixture def temp_workspace_dir(): """Create a temporary directory for test workspaces.""" with tempfile.TemporaryDirectory() as temp_dir: yield Path(temp_dir) @pytest.fixture def experimental_feature_flags(): """Enable experimental feature flags for tests.""" original_flags = constants.FEATURE_FLAG.copy() constants.FEATURE_FLAG.add("enable_experimental_features") constants.FEATURE_FLAG.add("enable_exclude_folder") constants.FEATURE_FLAG.add("enable_include_folder") constants.FEATURE_FLAG.add("enable_items_to_include") yield constants.FEATURE_FLAG.clear() constants.FEATURE_FLAG.update(original_flags) def create_test_item(base_path: Path, folder: Optional[str], name: str, item_type: str, logical_id: str) -> Path: """Helper to create a test item with .platform file. Args: base_path: Root directory for the workspace. folder: Subfolder path (e.g., "legacy" or "projects/team1") or None for root-level. name: Display name of the item. item_type: Type of the item (e.g., "Notebook", "SemanticModel"). logical_id: Logical ID for the item. Returns: Path to the created item directory. """ item_dir = base_path / folder / f"{name}.{item_type}" if folder else base_path / f"{name}.{item_type}" item_dir.mkdir(parents=True, exist_ok=True) platform_file = item_dir / ".platform" metadata = { "metadata": { "type": item_type, "displayName": name, "description": f"Test {item_type}", }, "config": {"logicalId": logical_id}, } with platform_file.open("w", encoding="utf-8") as f: json.dump(metadata, f) with (item_dir / "dummy.txt").open("w", encoding="utf-8") as f: f.write("Dummy file content") return item_dir # ============================================================================= # Basic Publishing Tests # ============================================================================= def test_publish_only_existing_item_types(mock_endpoint, temp_workspace_dir): """Test that publish_all_items only attempts to publish item types that exist in repository.""" create_test_item(temp_workspace_dir, None, "TestNotebook", "Notebook", "test-notebook-id") with ( patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), patch.object(FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", {})), patch.object( FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) ), patch("fabric_cicd._items._notebook.NotebookPublisher") as mock_notebook_cls, patch("fabric_cicd._items._environment.EnvironmentPublisher") as mock_env_cls, ): mock_notebook_instance = mock_notebook_cls.return_value workspace = FabricWorkspace( workspace_id="12345678-1234-5678-abcd-1234567890ab", repository_directory=str(temp_workspace_dir), token_credential=DummyTokenCredential(), ) publish.publish_all_items(workspace) assert "Notebook" in workspace.repository_items assert "Environment" not in workspace.repository_items mock_notebook_cls.assert_called_once_with(workspace) mock_notebook_instance.publish_all.assert_called_once() mock_env_cls.assert_not_called() def test_publish_ontology_item(mock_endpoint, temp_workspace_dir): """Test that publish_all_items publishes Ontology items when present in repository.""" create_test_item(temp_workspace_dir, None, "TestOntology", "Ontology", "test-ontology-id") with ( patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), patch.object(FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", {})), patch.object( FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) ), patch("fabric_cicd._items._ontology.OntologyPublisher") as mock_ontology_cls, ): mock_ontology_instance = mock_ontology_cls.return_value workspace = FabricWorkspace( workspace_id="12345678-1234-5678-abcd-1234567890ab", repository_directory=str(temp_workspace_dir), token_credential=DummyTokenCredential(), ) publish.publish_all_items(workspace) assert "Ontology" in workspace.repository_items mock_ontology_cls.assert_called_once_with(workspace) mock_ontology_instance.publish_all.assert_called_once() def test_publish_data_build_tool_job_item(mock_endpoint, temp_workspace_dir): """Test that publish_all_items publishes DataBuildToolJob items when present in repository.""" create_test_item(temp_workspace_dir, None, "TestDbtJob", "DataBuildToolJob", "test-dbt-job-id") with ( patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), patch.object(FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", {})), patch.object( FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) ), patch("fabric_cicd._items._databuildtooljob.DataBuildToolJobPublisher") as mock_dbt_job_cls, ): mock_dbt_job_instance = mock_dbt_job_cls.return_value workspace = FabricWorkspace( workspace_id="12345678-1234-5678-abcd-1234567890ab", repository_directory=str(temp_workspace_dir), token_credential=DummyTokenCredential(), ) publish.publish_all_items(workspace) assert "DataBuildToolJob" in workspace.repository_items mock_dbt_job_cls.assert_called_once_with(workspace) mock_dbt_job_instance.publish_all.assert_called_once() def test_default_none_item_type_in_scope_includes_all_types(mock_endpoint, temp_workspace_dir): """Test that when item_type_in_scope is None (default), all available item types are included.""" with ( patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), patch.object(FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", {})), patch.object( FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) ), ): workspace = FabricWorkspace( workspace_id="12345678-1234-5678-abcd-1234567890ab", repository_directory=str(temp_workspace_dir), token_credential=DummyTokenCredential(), ) expected_types = list(constants.ACCEPTED_ITEM_TYPES) assert set(workspace.item_type_in_scope) == set(expected_types) def test_empty_item_type_in_scope_list(mock_endpoint, temp_workspace_dir): """Test that passing an empty item_type_in_scope list works (no items to process).""" with patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint): workspace = FabricWorkspace( workspace_id="12345678-1234-5678-abcd-1234567890ab", repository_directory=str(temp_workspace_dir), item_type_in_scope=[], token_credential=DummyTokenCredential(), ) assert workspace.item_type_in_scope == [] # ============================================================================= # Invalid Item Type Tests # ============================================================================= def test_invalid_item_types_in_scope(mock_endpoint, temp_workspace_dir): """Test that passing invalid item types raises appropriate errors.""" with ( patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), pytest.raises(InputError, match="Invalid or unsupported item type: 'InvalidItemType'"), ): FabricWorkspace( workspace_id="12345678-1234-5678-abcd-1234567890ab", repository_directory=str(temp_workspace_dir), item_type_in_scope=["InvalidItemType"], token_credential=DummyTokenCredential(), ) def test_multiple_invalid_item_types_in_scope(mock_endpoint, temp_workspace_dir): """Test that passing multiple invalid item types raises error for the first invalid one.""" with ( patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), pytest.raises(InputError, match="Invalid or unsupported item type: 'FakeType'"), ): FabricWorkspace( workspace_id="12345678-1234-5678-abcd-1234567890ab", repository_directory=str(temp_workspace_dir), item_type_in_scope=["FakeType", "AnotherInvalidType"], token_credential=DummyTokenCredential(), ) def test_mixed_valid_and_invalid_item_types_in_scope(mock_endpoint, temp_workspace_dir): """Test that passing a mix of valid and invalid item types raises error for the invalid one.""" with ( patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), pytest.raises(InputError, match="Invalid or unsupported item type: 'BadType'"), ): FabricWorkspace( workspace_id="12345678-1234-5678-abcd-1234567890ab", repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook", "BadType", "Environment"], token_credential=DummyTokenCredential(), ) # ============================================================================= # Unpublish Feature Flag Tests # ============================================================================= def test_unpublish_feature_flag_warnings(mock_endpoint, temp_workspace_dir, caplog): """Test that warnings are logged when unpublish feature flags are missing.""" test_items = [ ("legacy", "TestLakehouse", "Lakehouse", "test-lakehouse-id"), ("legacy", "TestWarehouse", "Warehouse", "test-warehouse-id"), ("legacy", "TestSQLDB", "SQLDatabase", "test-sqldb-id"), ("legacy", "TestEventhouse", "Eventhouse", "test-eventhouse-id"), ] for folder, name, item_type, logical_id in test_items: create_test_item(temp_workspace_dir, folder, name, item_type, logical_id) deployed_items = {item_type: {name: MagicMock()} for _, name, item_type, _ in test_items} with ( patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), patch.object( FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", deployed_items), ), patch.object( FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) ), patch.object(FabricWorkspace, "_unpublish_folders", new=lambda _: None), caplog.at_level(logging.WARNING), ): workspace = FabricWorkspace( workspace_id="12345678-1234-5678-abcd-1234567890ab", repository_directory=str(temp_workspace_dir), item_type_in_scope=["Lakehouse", "Warehouse", "SQLDatabase", "Eventhouse"], token_credential=DummyTokenCredential(), ) publish.unpublish_all_orphan_items(workspace) expected_warnings = [ "Skipping unpublish for Lakehouse items because the 'enable_lakehouse_unpublish' feature flag is not enabled.", "Skipping unpublish for Warehouse items because the 'enable_warehouse_unpublish' feature flag is not enabled.", "Skipping unpublish for SQLDatabase items because the 'enable_sqldatabase_unpublish' feature flag is not enabled.", "Skipping unpublish for Eventhouse items because the 'enable_eventhouse_unpublish' feature flag is not enabled.", ] for expected_warning in expected_warnings: assert expected_warning in caplog.text def test_unpublish_with_feature_flags_enabled(mock_endpoint, temp_workspace_dir, caplog): """Test that no warnings are logged when unpublish feature flags are enabled.""" create_test_item(temp_workspace_dir, None, "TestLakehouse", "Lakehouse", "test-lakehouse-id") deployed_items = {"Lakehouse": {"TestLakehouse": MagicMock()}} original_flags = constants.FEATURE_FLAG.copy() constants.FEATURE_FLAG.add("enable_lakehouse_unpublish") try: with ( patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), patch.object( FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", deployed_items), ), patch.object( FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) ), patch.object(FabricWorkspace, "_unpublish_folders", new=lambda _: None), patch.object(FabricWorkspace, "_unpublish_item", new=lambda _, __, ___: None), caplog.at_level(logging.WARNING), ): workspace = FabricWorkspace( workspace_id="12345678-1234-5678-abcd-1234567890ab", repository_directory=str(temp_workspace_dir), item_type_in_scope=["Lakehouse"], token_credential=DummyTokenCredential(), ) publish.unpublish_all_orphan_items(workspace) assert "enable_lakehouse_unpublish" not in caplog.text assert "Skipping unpublish for Lakehouse" not in caplog.text finally: constants.FEATURE_FLAG.clear() constants.FEATURE_FLAG.update(original_flags) def test_unpublish_orphan_item_is_deleted(mock_endpoint, temp_workspace_dir): """Test that unpublish_all_orphan_items deletes an orphaned item not in the repository.""" create_test_item(temp_workspace_dir, None, "KeepMe", "Notebook", "keep-me-id") orphan_deployed = { "Notebook": { "KeepMe": MagicMock(guid="keep-guid"), "OrphanNotebook": MagicMock(guid="orphan-guid-123"), } } orphan_repo = {"Notebook": {"KeepMe": MagicMock()}} unpublish_calls = [] def track_unpublish(_self, item_name, item_type): unpublish_calls.append((item_name, item_type)) with ( patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), patch.object( FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", orphan_deployed), ), patch.object( FabricWorkspace, "_refresh_repository_items", new=lambda self: setattr(self, "repository_items", orphan_repo), ), patch.object( FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) ), patch.object(FabricWorkspace, "_unpublish_folders", new=lambda _: None), patch.object(FabricWorkspace, "_unpublish_item", new=track_unpublish), ): workspace = FabricWorkspace( workspace_id="12345678-1234-5678-abcd-1234567890ab", repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook"], token_credential=DummyTokenCredential(), ) publish.unpublish_all_orphan_items(workspace) assert len(unpublish_calls) == 1 assert unpublish_calls[0] == ("OrphanNotebook", "Notebook") def test_unpublish_orphan_excluded_by_regex(mock_endpoint, temp_workspace_dir): """Test that orphaned items matching the exclude regex are NOT unpublished.""" create_test_item(temp_workspace_dir, None, "KeepMe", "Notebook", "keep-me-id") orphan_deployed = { "Notebook": { "KeepMe": MagicMock(guid="keep-guid"), "ProtectedOrphan": MagicMock(guid="protected-guid"), "DeleteMe": MagicMock(guid="delete-guid"), } } orphan_repo = {"Notebook": {"KeepMe": MagicMock()}} unpublish_calls = [] def track_unpublish(_self, item_name, item_type): unpublish_calls.append((item_name, item_type)) with ( patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), patch.object( FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", orphan_deployed), ), patch.object( FabricWorkspace, "_refresh_repository_items", new=lambda self: setattr(self, "repository_items", orphan_repo), ), patch.object( FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) ), patch.object(FabricWorkspace, "_unpublish_folders", new=lambda _: None), patch.object(FabricWorkspace, "_unpublish_item", new=track_unpublish), ): workspace = FabricWorkspace( workspace_id="12345678-1234-5678-abcd-1234567890ab", repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook"], token_credential=DummyTokenCredential(), ) publish.unpublish_all_orphan_items(workspace, item_name_exclude_regex=r"^Protected.*") assert ("DeleteMe", "Notebook") in unpublish_calls assert ("ProtectedOrphan", "Notebook") not in unpublish_calls @pytest.mark.usefixtures("experimental_feature_flags") def test_unpublish_orphan_filtered_by_items_to_include(mock_endpoint, temp_workspace_dir): """Test that items_to_include limits which orphaned items are unpublished.""" create_test_item(temp_workspace_dir, None, "KeepMe", "Notebook", "keep-me-id") orphan_deployed = { "Notebook": { "KeepMe": MagicMock(guid="keep-guid"), "TargetOrphan": MagicMock(guid="target-guid"), "OtherOrphan": MagicMock(guid="other-guid"), } } orphan_repo = {"Notebook": {"KeepMe": MagicMock()}} unpublish_calls = [] def track_unpublish(_self, item_name, item_type): unpublish_calls.append((item_name, item_type)) with ( patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), patch.object( FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", orphan_deployed), ), patch.object( FabricWorkspace, "_refresh_repository_items", new=lambda self: setattr(self, "repository_items", orphan_repo), ), patch.object( FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) ), patch.object(FabricWorkspace, "_unpublish_folders", new=lambda _: None), patch.object(FabricWorkspace, "_unpublish_item", new=track_unpublish), ): workspace = FabricWorkspace( workspace_id="12345678-1234-5678-abcd-1234567890ab", repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook"], token_credential=DummyTokenCredential(), ) publish.unpublish_all_orphan_items(workspace, items_to_include=["TargetOrphan.Notebook"]) assert ("TargetOrphan", "Notebook") in unpublish_calls assert ("OtherOrphan", "Notebook") not in unpublish_calls def test_unpublish_no_orphans_no_deletion(mock_endpoint, temp_workspace_dir): """Test that unpublish_all_orphan_items does not call _unpublish_item when there are no orphans.""" create_test_item(temp_workspace_dir, None, "MyNotebook", "Notebook", "my-notebook-id") matching_items = {"Notebook": {"MyNotebook": MagicMock(guid="my-guid")}} unpublish_calls = [] def track_unpublish(_self, item_name, item_type): unpublish_calls.append((item_name, item_type)) with ( patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), patch.object( FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", matching_items), ), patch.object( FabricWorkspace, "_refresh_repository_items", new=lambda self: setattr(self, "repository_items", matching_items), ), patch.object( FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) ), patch.object(FabricWorkspace, "_unpublish_folders", new=lambda _: None), patch.object(FabricWorkspace, "_unpublish_item", new=track_unpublish), ): workspace = FabricWorkspace( workspace_id="12345678-1234-5678-abcd-1234567890ab", repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook"], token_credential=DummyTokenCredential(), ) publish.unpublish_all_orphan_items(workspace) assert len(unpublish_calls) == 0 # ============================================================================= # Publishing Order Tests # ============================================================================= def test_mirrored_database_published_before_lakehouse(mock_endpoint, temp_workspace_dir): """Test that MirroredDatabase items are published before Lakehouse items to enable shortcuts.""" call_order = [] def mock_publish_lakehouses(): call_order.append("Lakehouse") def mock_publish_mirroreddatabase(): call_order.append("MirroredDatabase") create_test_item(temp_workspace_dir, None, "TestLakehouse", "Lakehouse", "test-lakehouse-id") create_test_item(temp_workspace_dir, None, "TestMirroredDB", "MirroredDatabase", "test-mirrored-db-id") with ( patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), patch.object(FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", {})), patch.object( FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) ), patch("fabric_cicd._items._lakehouse.LakehousePublisher") as mock_lakehouse_cls, patch("fabric_cicd._items._mirroreddatabase.MirroredDatabasePublisher") as mock_mirrored_cls, ): mock_lakehouse_instance = mock_lakehouse_cls.return_value mock_lakehouse_instance.publish_all.side_effect = mock_publish_lakehouses mock_mirrored_instance = mock_mirrored_cls.return_value mock_mirrored_instance.publish_all.side_effect = mock_publish_mirroreddatabase workspace = FabricWorkspace( workspace_id="12345678-1234-5678-abcd-1234567890ab", repository_directory=str(temp_workspace_dir), item_type_in_scope=["Lakehouse", "MirroredDatabase"], token_credential=DummyTokenCredential(), ) publish.publish_all_items(workspace) assert len(call_order) == 2 assert "MirroredDatabase" in call_order assert "Lakehouse" in call_order mirrored_db_index = call_order.index("MirroredDatabase") lakehouse_index = call_order.index("Lakehouse") assert mirrored_db_index < lakehouse_index, ( f"MirroredDatabase should be published before Lakehouse, but got order: {call_order}" ) # ============================================================================= # Folder Exclusion Tests # ============================================================================= @pytest.mark.usefixtures("experimental_feature_flags") def test_folder_exclusion_with_regex(mock_endpoint, temp_workspace_dir): """Test that folder_path_exclude_regex can exclude entire folders of items.""" create_test_item(temp_workspace_dir, "legacy", "LegacyNotebook", "Notebook", "legacy-notebook-id") create_test_item(temp_workspace_dir, "legacy", "LegacyModel", "SemanticModel", "legacy-model-id") create_test_item(temp_workspace_dir, "current", "CurrentNotebook", "Notebook", "current-notebook-id") create_test_item(temp_workspace_dir, None, "RootNotebook", "Notebook", "root-notebook-id") with ( patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), patch.object(FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", {})), patch.object( FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) ), ): workspace = FabricWorkspace( workspace_id="12345678-1234-5678-abcd-1234567890ab", repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook", "SemanticModel"], token_credential=DummyTokenCredential(), ) exclude_regex = r".*legacy.*" publish.publish_all_items(workspace, folder_path_exclude_regex=exclude_regex) assert "Notebook" in workspace.repository_items assert "SemanticModel" in workspace.repository_items assert workspace.repository_items["Notebook"]["LegacyNotebook"].skip_publish is True assert workspace.repository_items["SemanticModel"]["LegacyModel"].skip_publish is True assert workspace.repository_items["Notebook"]["CurrentNotebook"].skip_publish is False assert workspace.repository_items["Notebook"]["RootNotebook"].skip_publish is False @pytest.mark.usefixtures("experimental_feature_flags") def test_folder_exclusion_with_anchored_regex(mock_endpoint, temp_workspace_dir): """Test that excluding a parent folder with an anchored regex also excludes items in child folders, preserving consistent hierarchy behavior.""" create_test_item(temp_workspace_dir, "legacy", "LegacyNotebook", "Notebook", "legacy-notebook-id") create_test_item(temp_workspace_dir, "legacy/archived", "ArchivedNotebook", "Notebook", "archived-notebook-id") create_test_item(temp_workspace_dir, "current", "CurrentNotebook", "Notebook", "current-notebook-id") with ( patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), patch.object(FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", {})), patch.object( FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) ), ): workspace = FabricWorkspace( workspace_id="12345678-1234-5678-abcd-1234567890ab", repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook"], token_credential=DummyTokenCredential(), ) exclude_regex = r"^/legacy$" publish.publish_all_items(workspace, folder_path_exclude_regex=exclude_regex) assert workspace.repository_items["Notebook"]["LegacyNotebook"].skip_publish is True assert workspace.repository_items["Notebook"]["ArchivedNotebook"].skip_publish is True assert workspace.repository_items["Notebook"]["CurrentNotebook"].skip_publish is False def test_item_name_exclusion_still_works(mock_endpoint, temp_workspace_dir): """Test that existing item name exclusion still works with the new folder exclusion feature.""" create_test_item(temp_workspace_dir, None, "TestNotebook", "Notebook", "test-notebook-id") create_test_item(temp_workspace_dir, None, "DoNotPublish", "Notebook", "excluded-notebook-id") with ( patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), patch.object(FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", {})), patch.object( FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) ), ): workspace = FabricWorkspace( workspace_id="12345678-1234-5678-abcd-1234567890ab", repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook"], token_credential=DummyTokenCredential(), ) exclude_regex = r".*DoNotPublish.*" publish.publish_all_items(workspace, item_name_exclude_regex=exclude_regex) assert workspace.repository_items["Notebook"]["DoNotPublish"].skip_publish is True assert workspace.repository_items["Notebook"]["TestNotebook"].skip_publish is False # ============================================================================= # Folder Inclusion Tests # ============================================================================= @pytest.mark.usefixtures("experimental_feature_flags") def test_folder_inclusion_with_folder_path_to_include(mock_endpoint, temp_workspace_dir): """Test that folder_path_to_include only filters items found within a Fabric folder.""" create_test_item(temp_workspace_dir, "active", "ActiveNotebook", "Notebook", "active-notebook-id") create_test_item(temp_workspace_dir, "active", "ActiveModel", "SemanticModel", "active-model-id") create_test_item(temp_workspace_dir, "archive", "ArchivedNotebook", "Notebook", "archived-notebook-id") create_test_item(temp_workspace_dir, None, "RootNotebook", "Notebook", "root-notebook-id") create_test_item(temp_workspace_dir, "projects", "ProjectNotebook", "Notebook", "projects-notebook-id") create_test_item(temp_workspace_dir, "projects/team1", "NestedNotebook", "Notebook", "nested-notebook-id") create_test_item(temp_workspace_dir, "dept", "DeptNotebook", "Notebook", "dept-notebook-id") create_test_item(temp_workspace_dir, "dept/eng", "EngNotebook", "Notebook", "eng-notebook-id") with ( patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), patch.object(FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", {})), patch.object( FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) ), ): workspace = FabricWorkspace( workspace_id="12345678-1234-5678-abcd-1234567890ab", repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook", "SemanticModel"], token_credential=DummyTokenCredential(), ) publish.publish_all_items( workspace, folder_path_to_include=["/active", "/projects/team1", "/dept", "/dept/eng"], ) assert "Notebook" in workspace.repository_items assert "SemanticModel" in workspace.repository_items assert workspace.repository_items["Notebook"]["ActiveNotebook"].skip_publish is False assert workspace.repository_items["SemanticModel"]["ActiveModel"].skip_publish is False assert workspace.repository_items["Notebook"]["ArchivedNotebook"].skip_publish is True assert workspace.repository_items["Notebook"]["RootNotebook"].skip_publish is False assert workspace.repository_items["Notebook"]["NestedNotebook"].skip_publish is False assert workspace.repository_items["Notebook"]["ProjectNotebook"].skip_publish is True assert workspace.repository_items["Notebook"]["DeptNotebook"].skip_publish is False assert workspace.repository_items["Notebook"]["EngNotebook"].skip_publish is False @pytest.mark.usefixtures("experimental_feature_flags") def test_folder_inclusion_and_exclusion_together(mock_endpoint, temp_workspace_dir): """Test that using both folder_path_to_include and folder_path_exclude_regex raises InputError.""" create_test_item(temp_workspace_dir, "deploy", "DeployNotebook", "Notebook", "deploy-notebook-id") with ( patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), patch.object(FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", {})), patch.object( FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) ), ): workspace = FabricWorkspace( workspace_id="12345678-1234-5678-abcd-1234567890ab", repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook"], token_credential=DummyTokenCredential(), ) with pytest.raises( InputError, match="Cannot use both 'folder_path_exclude_regex' and 'folder_path_to_include'", ): publish.publish_all_items( workspace, folder_path_to_include=["/deploy"], folder_path_exclude_regex=r"^/deploy/legacy", ) @pytest.mark.usefixtures("experimental_feature_flags") def test_empty_folder_path_to_include_raises_error(mock_endpoint, temp_workspace_dir): """Test that passing an empty list for folder_path_to_include raises an InputError.""" with ( patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), patch.object(FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", {})), patch.object( FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) ), ): workspace = FabricWorkspace( workspace_id="12345678-1234-5678-abcd-1234567890ab", repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook"], token_credential=DummyTokenCredential(), ) with pytest.raises(InputError, match="folder_path_to_include must not be an empty list"): publish.publish_all_items(workspace, folder_path_to_include=[]) # ============================================================================= # Combined Filter Tests # ============================================================================= @pytest.mark.usefixtures("experimental_feature_flags") def test_folder_exclusion_with_items_to_include(mock_endpoint, temp_workspace_dir): """Test that folder exclusion takes precedence over items_to_include.""" create_test_item(temp_workspace_dir, "legacy", "ImportantNotebook", "Notebook", "important-notebook-id") create_test_item(temp_workspace_dir, None, "StandaloneNotebook", "Notebook", "standalone-notebook-id") create_test_item(temp_workspace_dir, None, "OtherNotebook", "Notebook", "other-notebook-id") with ( patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), patch.object(FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", {})), patch.object( FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) ), ): workspace = FabricWorkspace( workspace_id="12345678-1234-5678-abcd-1234567890ab", repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook"], token_credential=DummyTokenCredential(), ) publish.publish_all_items( workspace, folder_path_exclude_regex=r"^/legacy", items_to_include=["ImportantNotebook.Notebook", "StandaloneNotebook.Notebook"], ) assert workspace.repository_items["Notebook"]["ImportantNotebook"].skip_publish is True assert workspace.repository_items["Notebook"]["StandaloneNotebook"].skip_publish is False # OtherNotebook is excluded by get_items_to_publish() because it is not in # items_to_include, so publish_all() marks it skip_publish=True. assert workspace.repository_items["Notebook"]["OtherNotebook"].skip_publish is True @pytest.mark.usefixtures("experimental_feature_flags") def test_folder_inclusion_with_item_exclusion(mock_endpoint, temp_workspace_dir): """Test that item_name_exclude_regex can exclude specific items within an included folder.""" create_test_item(temp_workspace_dir, "active", "ActiveNotebook", "Notebook", "active-notebook-id") create_test_item(temp_workspace_dir, "active", "DebugNotebook", "Notebook", "debug-notebook-id") with ( patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), patch.object(FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", {})), patch.object( FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) ), ): workspace = FabricWorkspace( workspace_id="12345678-1234-5678-abcd-1234567890ab", repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook"], token_credential=DummyTokenCredential(), ) publish.publish_all_items( workspace, folder_path_to_include=["/active"], item_name_exclude_regex=r"^Debug.*", ) assert workspace.repository_items["Notebook"]["DebugNotebook"].skip_publish is True assert workspace.repository_items["Notebook"]["ActiveNotebook"].skip_publish is False @pytest.mark.usefixtures("experimental_feature_flags") def test_folder_inclusion_with_items_to_include(mock_endpoint, temp_workspace_dir): """Test that folder_path_to_include and items_to_include work together to narrow the scope.""" create_test_item(temp_workspace_dir, "active", "Notebook1", "Notebook", "notebook1-id") create_test_item(temp_workspace_dir, "active", "Notebook2", "Notebook", "notebook2-id") create_test_item(temp_workspace_dir, "archive", "ArchivedNotebook", "Notebook", "archived-notebook-id") with ( patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), patch.object(FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", {})), patch.object( FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) ), ): workspace = FabricWorkspace( workspace_id="12345678-1234-5678-abcd-1234567890ab", repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook"], token_credential=DummyTokenCredential(), ) publish.publish_all_items( workspace, folder_path_to_include=["/active"], items_to_include=["Notebook1.Notebook"], ) assert workspace.repository_items["Notebook"]["Notebook1"].skip_publish is False # Notebook2 and ArchivedNotebook are excluded by get_items_to_publish() # because they are not in items_to_include, so publish_all() marks them skip_publish=True. assert workspace.repository_items["Notebook"]["Notebook2"].skip_publish is True assert workspace.repository_items["Notebook"]["ArchivedNotebook"].skip_publish is True @pytest.mark.usefixtures("experimental_feature_flags") def test_all_filters_combined(mock_endpoint, temp_workspace_dir): """Test the complete filter evaluation order with all filters applied.""" create_test_item(temp_workspace_dir, "active", "DebugNotebook", "Notebook", "debug-id") create_test_item(temp_workspace_dir, "active", "TargetNotebook", "Notebook", "target-id") create_test_item(temp_workspace_dir, "active", "OtherNotebook", "Notebook", "other-id") create_test_item(temp_workspace_dir, "archive", "ArchivedNotebook", "Notebook", "archive-id") with ( patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), patch.object(FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", {})), patch.object( FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) ), ): workspace = FabricWorkspace( workspace_id="12345678-1234-5678-abcd-1234567890ab", repository_directory=str(temp_workspace_dir), item_type_in_scope=["Notebook"], token_credential=DummyTokenCredential(), ) publish.publish_all_items( workspace, item_name_exclude_regex=r"^Debug.*", folder_path_to_include=["/active"], items_to_include=["TargetNotebook.Notebook"], ) # DebugNotebook, OtherNotebook, and ArchivedNotebook are excluded by # get_items_to_publish() because they are not in items_to_include, so # publish_all() marks them skip_publish=True. assert workspace.repository_items["Notebook"]["DebugNotebook"].skip_publish is True assert workspace.repository_items["Notebook"]["TargetNotebook"].skip_publish is False assert workspace.repository_items["Notebook"]["OtherNotebook"].skip_publish is True assert workspace.repository_items["Notebook"]["ArchivedNotebook"].skip_publish is True # ============================================================================= # NotebookPublisher Tests # ============================================================================= class TestNotebookPublisher: """Tests for NotebookPublisher.publish_one method.""" @pytest.fixture def mock_workspace(self): """Create a mock FabricWorkspace object.""" workspace = MagicMock() workspace._publish_item = MagicMock() return workspace @pytest.fixture def publisher(self, mock_workspace): """Create a NotebookPublisher instance.""" publisher = NotebookPublisher.__new__(NotebookPublisher) publisher.fabric_workspace_obj = mock_workspace return publisher def _create_mock_item(self, file_suffix: str) -> MagicMock: """Create a mock Item with a file of the given suffix.""" mock_file = MagicMock() mock_file.file_path = Path(f"notebook{file_suffix}") mock_item = MagicMock() mock_item.item_files = [mock_file] return mock_item def test_publish_ipynb_includes_api_format(self, publisher, mock_workspace): """Test that .ipynb files include api_format in kwargs.""" item = self._create_mock_item(".ipynb") publisher.publish_one("test_notebook", item) expected_api_format = API_FORMAT_MAPPING.get(ItemType.NOTEBOOK.value) mock_workspace._publish_item.assert_called_once_with( item_name="test_notebook", item_type=ItemType.NOTEBOOK.value, api_format=expected_api_format, ) def test_publish_non_ipynb_excludes_api_format(self, publisher, mock_workspace): """Test that non-.ipynb files do not include api_format.""" item = self._create_mock_item(".py") publisher.publish_one("test_notebook", item) mock_workspace._publish_item.assert_called_once_with( item_name="test_notebook", item_type=ItemType.NOTEBOOK.value, ) def test_publish_mixed_files_with_ipynb(self, publisher, mock_workspace): """Test that if any file is .ipynb, api_format is included.""" mock_file_py = MagicMock() mock_file_py.file_path = Path("script.py") mock_file_ipynb = MagicMock() mock_file_ipynb.file_path = Path("notebook.ipynb") mock_item = MagicMock() mock_item.item_files = [mock_file_py, mock_file_ipynb] publisher.publish_one("test_notebook", mock_item) expected_api_format = API_FORMAT_MAPPING.get(ItemType.NOTEBOOK.value) mock_workspace._publish_item.assert_called_once_with( item_name="test_notebook", item_type=ItemType.NOTEBOOK.value, api_format=expected_api_format, ) def test_item_type_is_notebook(self, publisher): """Test that item_type is correctly set to Notebook.""" assert publisher.item_type == ItemType.NOTEBOOK.value def test_files_sorted_same_stem_content_before_settings(self, publisher): """Test content file precedes settings even when filenames share the same stem.""" mock_platform = MagicMock() mock_platform.file_path = Path(".platform") mock_settings = MagicMock() mock_settings.file_path = Path("notebook-settings.json") mock_content = MagicMock() mock_content.file_path = Path("notebook-content.py") mock_item = MagicMock() mock_item.item_files = [mock_settings, mock_content, mock_platform] publisher.publish_one("test_notebook", mock_item) assert mock_item.item_files[0].file_path.name == ".platform" assert mock_item.item_files[1].file_path.name == "notebook-content.py" assert mock_item.item_files[2].file_path.name == "notebook-settings.json" def test_files_sorted_with_unknown_extension(self, publisher): """Test that unknown file extensions get default priority (2) between content and settings.""" mock_platform = MagicMock() mock_platform.file_path = Path(".platform") mock_settings = MagicMock() mock_settings.file_path = Path("notebook.json") mock_content = MagicMock() mock_content.file_path = Path("notebook.py") mock_other = MagicMock() mock_other.file_path = Path("readme.md") # Unknown extension mock_item = MagicMock() mock_item.item_files = [mock_settings, mock_other, mock_content, mock_platform] publisher.publish_one("test_notebook", mock_item) # Expected order: .platform (0), notebook.py (1), readme.md (2), notebook.json (3) assert mock_item.item_files[0].file_path.name == ".platform" assert mock_item.item_files[1].file_path.name == "notebook.py" assert mock_item.item_files[2].file_path.name == "readme.md" assert mock_item.item_files[3].file_path.name == "notebook.json" ================================================ FILE: tests/test_response_collection.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Test response collection functionality.""" import json import tempfile from pathlib import Path from unittest.mock import MagicMock, patch import pytest from fixtures.credentials import DummyTokenCredential import fabric_cicd.constants as constants import fabric_cicd.publish as publish from fabric_cicd import append_feature_flag from fabric_cicd.fabric_workspace import FabricWorkspace @pytest.fixture def mock_endpoint(): """Mock FabricEndpoint to return realistic responses.""" mock = MagicMock() def mock_invoke(method, url, body=None, **_kwargs): if method == "GET" and "workspaces" in url and not url.endswith("/items"): return {"body": {"value": [], "capacityId": "test-capacity"}} if method == "GET" and url.endswith("/items"): return {"body": {"value": []}} if method == "POST" and url.endswith("/folders"): return {"body": {"id": "mock-folder-id"}} if method == "POST" and url.endswith("/items"): return { "body": { "id": "mock-item-id-12345", "workspaceId": "mock-workspace-id", "displayName": body.get("displayName", "Test Item"), "type": body.get("type", "Notebook"), } } if method == "POST" and "updateDefinition" in url: return {"body": {"message": "Definition updated successfully"}} if method == "PATCH" and "items/" in url: return {"body": {"message": "Item metadata updated successfully"}} if method == "POST" and url.endswith("/move"): return {"body": {"message": "Item moved successfully"}} if method == "DELETE" and "items/" in url: return {"body": {}, "header": {}, "status_code": 200} return {"body": {"value": [], "capacityId": "test-capacity"}} mock.invoke.side_effect = mock_invoke return mock @pytest.fixture def test_workspace_with_notebook(mock_endpoint): """Create a test workspace with a notebook item.""" with tempfile.TemporaryDirectory() as temp_dir: temp_path = Path(temp_dir) # Create a notebook item notebook_dir = temp_path / "TestNotebook.Notebook" notebook_dir.mkdir(parents=True, exist_ok=True) platform_file = notebook_dir / ".platform" platform_file.write_text( json.dumps({ "metadata": { "kernel_info": {"name": "synapse_pyspark"}, "language_info": {"name": "python"}, } }) ) notebook_file = notebook_dir / "notebook-content.py" notebook_file.write_text("# Test notebook content\nprint('Hello World')") # Patch FabricEndpoint before creating workspace with ( patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), patch.object( FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", {}) ), patch.object( FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) ), patch.object(FabricWorkspace, "_refresh_repository_items", new=lambda _: None), patch.object(FabricWorkspace, "_refresh_repository_folders", new=lambda _: None), ): workspace = FabricWorkspace( workspace_id="12345678-1234-5678-abcd-1234567890ab", repository_directory=str(temp_path), item_type_in_scope=["Notebook"], token_credential=DummyTokenCredential(), ) # Manually set up repository items since we're patching the refresh methods workspace.repository_items = { "Notebook": { "TestNotebook": MagicMock( guid=None, folder_id="mock-folder-id", logical_id="test-notebook-logical-id", item_files=[ MagicMock( relative_path="notebook-content.py", type="text", file_path=notebook_file, contents="# Test notebook content\nprint('Hello World')", base64_payload={"path": "notebook-content.py", "payloadType": "InlineBase64"}, ) ], skip_publish=False, path=notebook_dir, ) } } workspace.deployed_items = {} # Set up parameter data to avoid parameter file warnings workspace.parameter_data = {} workspace.parameter_file_path = None yield workspace # ============================================================================= # Initialization Tests # ============================================================================= def test_responses_initialized_as_none(test_workspace_with_notebook): """Test that responses and unpublish_responses attributes are initialized as None by default.""" workspace = test_workspace_with_notebook assert workspace.responses is None assert workspace.unpublish_responses is None # ============================================================================= # Publish Response Collection Tests # ============================================================================= def test_publish_item_without_response_collection(test_workspace_with_notebook): """Test that _publish_item works normally when responses is None.""" workspace = test_workspace_with_notebook with ( patch.object(workspace, "_replace_logical_ids", side_effect=lambda x: x), patch.object(workspace, "_replace_parameters", side_effect=lambda file, _: file.contents), patch.object(workspace, "_replace_workspace_ids", side_effect=lambda x: x), ): workspace._publish_item(item_name="TestNotebook", item_type="Notebook") assert workspace.responses is None assert workspace.unpublish_responses is None def test_publish_item_with_response_collection(test_workspace_with_notebook): """Test that _publish_item stores responses when feature flag is enabled.""" workspace = test_workspace_with_notebook constants.FEATURE_FLAG.add("enable_response_collection") try: workspace.responses = {} with ( patch.object(workspace, "_replace_logical_ids", side_effect=lambda x: x), patch.object(workspace, "_replace_parameters", side_effect=lambda file, _: file.contents), patch.object(workspace, "_replace_workspace_ids", side_effect=lambda x: x), ): workspace._publish_item(item_name="TestNotebook", item_type="Notebook") assert workspace.responses is not None assert "Notebook" in workspace.responses assert "TestNotebook" in workspace.responses["Notebook"] response = workspace.responses["Notebook"]["TestNotebook"] assert response["body"]["id"] == "mock-item-id-12345" finally: constants.FEATURE_FLAG.discard("enable_response_collection") def test_publish_all_items_no_feature_flag(test_workspace_with_notebook): """Test that publish_all_items doesn't enable responses by default.""" workspace = test_workspace_with_notebook result = publish.publish_all_items(workspace) assert result is None assert workspace.responses is None def test_publish_all_items_with_feature_flag(test_workspace_with_notebook): """Test that publish_all_items enables response collection when feature flag is set.""" workspace = test_workspace_with_notebook constants.FEATURE_FLAG.add("enable_response_collection") try: result = publish.publish_all_items(workspace) assert workspace.responses is not None assert isinstance(workspace.responses, dict) assert result is workspace.responses finally: constants.FEATURE_FLAG.discard("enable_response_collection") def test_workspace_responses_access_pattern(test_workspace_with_notebook): """Test the recommended access pattern for responses.""" workspace = test_workspace_with_notebook constants.FEATURE_FLAG.add("enable_response_collection") try: publish.publish_all_items(workspace) assert hasattr(workspace, "responses") assert workspace.responses is not None if workspace.responses: for item_type, items in workspace.responses.items(): assert isinstance(item_type, str) assert isinstance(items, dict) for item_name, response in items.items(): assert isinstance(item_name, str) assert isinstance(response, dict) finally: constants.FEATURE_FLAG.discard("enable_response_collection") def test_publish_item_skipped_no_response_stored(test_workspace_with_notebook): """Test that skipped items don't store responses even when collection is enabled.""" workspace = test_workspace_with_notebook constants.FEATURE_FLAG.add("enable_response_collection") try: workspace.responses = {} workspace.publish_item_name_exclude_regex = "TestNotebook" workspace._publish_item(item_name="TestNotebook", item_type="Notebook") assert "Notebook" not in workspace.responses finally: constants.FEATURE_FLAG.discard("enable_response_collection") def test_append_feature_flag_enables_response_collection(test_workspace_with_notebook): """Test that using append_feature_flag enables response collection.""" workspace = test_workspace_with_notebook append_feature_flag("enable_response_collection") try: result = publish.publish_all_items(workspace) assert workspace.responses is not None assert isinstance(workspace.responses, dict) assert result is workspace.responses finally: constants.FEATURE_FLAG.discard("enable_response_collection") # ============================================================================= # Unpublish Response Collection Tests # ============================================================================= def test_unpublish_item_without_response_collection(test_workspace_with_notebook): """Test that _unpublish_item does not store responses when collection is disabled.""" workspace = test_workspace_with_notebook workspace.deployed_items = {"Notebook": {"TestNotebook": MagicMock(guid="mock-guid-123")}} workspace._unpublish_item(item_name="TestNotebook", item_type="Notebook") assert workspace.unpublish_responses is None def test_unpublish_item_with_response_collection(test_workspace_with_notebook): """Test that _unpublish_item stores responses in unpublish_responses.""" workspace = test_workspace_with_notebook workspace.deployed_items = {"Notebook": {"TestNotebook": MagicMock(guid="mock-guid-123")}} constants.FEATURE_FLAG.add("enable_response_collection") try: workspace.unpublish_responses = {} workspace._unpublish_item(item_name="TestNotebook", item_type="Notebook") assert "Notebook" in workspace.unpublish_responses assert "TestNotebook" in workspace.unpublish_responses["Notebook"] finally: constants.FEATURE_FLAG.discard("enable_response_collection") def test_unpublish_item_does_not_write_to_publish_responses(test_workspace_with_notebook): """Test that _unpublish_item does not write to self.responses.""" workspace = test_workspace_with_notebook workspace.deployed_items = {"Notebook": {"TestNotebook": MagicMock(guid="mock-guid-123")}} constants.FEATURE_FLAG.add("enable_response_collection") try: workspace.responses = {"Notebook": {"ExistingItem": {"body": {"id": "existing"}}}} workspace.unpublish_responses = {} workspace._unpublish_item(item_name="TestNotebook", item_type="Notebook") # publish responses unchanged — no TestNotebook added assert "TestNotebook" not in workspace.responses.get("Notebook", {}) assert workspace.responses["Notebook"]["ExistingItem"]["body"]["id"] == "existing" finally: constants.FEATURE_FLAG.discard("enable_response_collection") def test_unpublish_item_failure_does_not_store_response(test_workspace_with_notebook, mock_endpoint): """Test that _unpublish_item does not store responses when the DELETE call fails.""" workspace = test_workspace_with_notebook workspace.deployed_items = {"Notebook": {"TestNotebook": MagicMock(guid="mock-guid-123")}} constants.FEATURE_FLAG.add("enable_response_collection") # Make DELETE raise an exception original_side_effect = mock_endpoint.invoke.side_effect def failing_invoke(method, url, **kwargs): if method == "DELETE": msg = "API error" raise Exception(msg) return original_side_effect(method, url, **kwargs) mock_endpoint.invoke.side_effect = failing_invoke try: workspace.unpublish_responses = {} workspace._unpublish_item(item_name="TestNotebook", item_type="Notebook") # No response stored due to failure assert "Notebook" not in workspace.unpublish_responses finally: mock_endpoint.invoke.side_effect = original_side_effect constants.FEATURE_FLAG.discard("enable_response_collection") def test_unpublish_all_orphan_items_no_feature_flag(test_workspace_with_notebook): """Test that unpublish_all_orphan_items returns None without the feature flag.""" workspace = test_workspace_with_notebook workspace.deployed_items = {} result = publish.unpublish_all_orphan_items(workspace) assert result is None assert workspace.unpublish_responses is None def test_unpublish_all_orphan_items_with_feature_flag(test_workspace_with_notebook): """Test that unpublish_all_orphan_items initializes unpublish_responses and returns populated dict.""" workspace = test_workspace_with_notebook # Set up an orphaned item: deployed but not in repository orphan_deployed = {"Notebook": {"OrphanNotebook": MagicMock(guid="orphan-guid-456")}} orphan_repo = {} constants.FEATURE_FLAG.add("enable_response_collection") try: assert workspace.unpublish_responses is None with ( patch.object( FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", orphan_deployed), ), patch.object( FabricWorkspace, "_refresh_repository_items", new=lambda self: setattr(self, "repository_items", orphan_repo), ), ): result = publish.unpublish_all_orphan_items(workspace) assert workspace.unpublish_responses is not None assert isinstance(workspace.unpublish_responses, dict) assert "Notebook" in workspace.unpublish_responses assert "OrphanNotebook" in workspace.unpublish_responses["Notebook"] assert result is workspace.unpublish_responses finally: constants.FEATURE_FLAG.discard("enable_response_collection") def test_unpublish_all_orphan_items_empty_returns_none(test_workspace_with_notebook): """Test that unpublish_all_orphan_items returns None when no items are orphaned.""" workspace = test_workspace_with_notebook workspace.deployed_items = {} constants.FEATURE_FLAG.add("enable_response_collection") try: result = publish.unpublish_all_orphan_items(workspace) # Empty responses dict is falsy, so return value is None assert result is None assert workspace.unpublish_responses is not None assert isinstance(workspace.unpublish_responses, dict) finally: constants.FEATURE_FLAG.discard("enable_response_collection") # ============================================================================= # Publish and Unpublish Response Separation Tests # ============================================================================= def test_unpublish_does_not_modify_publish_responses(test_workspace_with_notebook): """Test that unpublish_all_orphan_items does not modify publish responses.""" workspace = test_workspace_with_notebook workspace.deployed_items = {} constants.FEATURE_FLAG.add("enable_response_collection") try: workspace.responses = {"Notebook": {"TestNotebook": {"body": {"id": "publish-response"}}}} publish.unpublish_all_orphan_items(workspace) # Publish responses untouched assert workspace.responses["Notebook"]["TestNotebook"]["body"]["id"] == "publish-response" # Unpublish responses initialized separately assert workspace.unpublish_responses is not None assert isinstance(workspace.unpublish_responses, dict) finally: constants.FEATURE_FLAG.discard("enable_response_collection") def test_publish_does_not_modify_unpublish_responses(test_workspace_with_notebook): """Test that publish_all_items does not modify unpublish responses.""" workspace = test_workspace_with_notebook constants.FEATURE_FLAG.add("enable_response_collection") try: workspace.unpublish_responses = {"Notebook": {"OldNotebook": {"body": {"id": "unpublish-response"}}}} publish.publish_all_items(workspace) # Unpublish responses untouched assert workspace.unpublish_responses["Notebook"]["OldNotebook"]["body"]["id"] == "unpublish-response" # Publish responses initialized separately assert workspace.responses is not None assert isinstance(workspace.responses, dict) finally: constants.FEATURE_FLAG.discard("enable_response_collection") def test_publish_and_unpublish_responses_are_separate_dicts(test_workspace_with_notebook): """Test that publish and unpublish use separate response dictionaries.""" workspace = test_workspace_with_notebook workspace.deployed_items = {} constants.FEATURE_FLAG.add("enable_response_collection") try: publish.publish_all_items(workspace) publish.unpublish_all_orphan_items(workspace) assert workspace.responses is not workspace.unpublish_responses finally: constants.FEATURE_FLAG.discard("enable_response_collection") ================================================ FILE: tests/test_semantic_model_exclude.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """ Regression tests: items_to_include + semantic model connection binding. When items_to_include scopes to a subset of semantic models, excluded models have skip_publish=True and guid="" after publish_all(). bind_semanticmodel_to_connection() must not attempt to bind them — doing so would produce URLs like GET items//connections POST semanticModels//bindConnection which return HTTP 400. """ from unittest.mock import MagicMock from fabric_cicd._common._item import Item from fabric_cicd._items._semanticmodel import SemanticModelPublisher, bind_semanticmodel_to_connection from fabric_cicd.fabric_workspace import FabricWorkspace # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def _make_sm(name: str, guid: str = "", skip_publish: bool = False) -> Item: """Create a real SemanticModel Item for testing.""" item = Item(type="SemanticModel", name=name, description="", guid=guid) item.skip_publish = skip_publish return item def _make_connections(*conn_ids: str) -> dict: """Build a minimal connections dict keyed by connection ID.""" return { cid: {"id": cid, "connectivityType": "ShareableCloud", "connectionDetails": {"type": "SQL", "path": "srv"}} for cid in conn_ids } # --------------------------------------------------------------------------- # Unit tests for bind_semanticmodel_to_connection() # --------------------------------------------------------------------------- def test_bind_skips_model_with_skip_publish_true(): """ bind_semanticmodel_to_connection() must skip models whose skip_publish=True. No HTTP call should be made for the excluded model. """ included = _make_sm("SalesModel", guid="sales-guid", skip_publish=False) excluded = _make_sm("DevModel", guid="", skip_publish=True) # excluded via items_to_include workspace = MagicMock(spec=FabricWorkspace) workspace.workspace_id = "ws-123" workspace.endpoint = MagicMock() workspace.repository_items = {"SemanticModel": {"SalesModel": included, "DevModel": excluded}} workspace.endpoint.invoke.return_value = { "body": {"value": [{"id": "old-conn", "connectivityType": "ShareableCloud", "connectionDetails": {}}]}, "status_code": 200, } connections = _make_connections("conn-001") connection_details = {"SalesModel": "conn-001", "DevModel": "conn-001"} bind_semanticmodel_to_connection(workspace, connections, connection_details) called_urls = [c[1]["url"] for c in workspace.endpoint.invoke.call_args_list] # Positive assertion: SalesModel MUST have triggered API calls (guards against vacuous all([])) assert any("sales-guid" in url for url in called_urls), ( f"SalesModel should have triggered API calls, got: {called_urls}" ) # Negative assertion: DevModel must be entirely skipped assert not any("DevModel" in url for url in called_urls), ( f"Empty-GUID or DevModel URL must not be called, got: {called_urls}" ) # Extra guard: no empty-segment URLs at all assert not any("//" in url.split("://", 1)[-1] for url in called_urls), ( f"Empty-GUID URL must not appear, got: {called_urls}" ) def test_bind_skips_model_without_guid(): """ bind_semanticmodel_to_connection() must skip models whose guid is empty, even when skip_publish is False. This is a defensive safety net against any future code path that leaves skip_publish unset. """ deployed = _make_sm("SalesModel", guid="sales-guid", skip_publish=False) not_deployed = _make_sm("DevModel", guid="", skip_publish=False) # guid guard must catch this workspace = MagicMock(spec=FabricWorkspace) workspace.workspace_id = "ws-123" workspace.endpoint = MagicMock() workspace.repository_items = {"SemanticModel": {"SalesModel": deployed, "DevModel": not_deployed}} workspace.endpoint.invoke.return_value = { "body": {"value": [{"id": "old-conn", "connectivityType": "ShareableCloud", "connectionDetails": {}}]}, "status_code": 200, } connections = _make_connections("conn-001") connection_details = {"SalesModel": "conn-001", "DevModel": "conn-001"} bind_semanticmodel_to_connection(workspace, connections, connection_details) called_urls = [c[1]["url"] for c in workspace.endpoint.invoke.call_args_list] # An empty GUID produces a path like "items//connections"; strip the scheme to detect it. assert not any("//" in url.split("://", 1)[-1] for url in called_urls), ( f"Empty-GUID URL must not appear, got: {called_urls}" ) assert not any("DevModel" in url for url in called_urls), f"DevModel must be skipped, got: {called_urls}" def test_bind_processes_included_models_normally(): """ Models with skip_publish=False and a valid guid must still be bound. """ model_a = _make_sm("ModelA", guid="guid-a", skip_publish=False) model_b = _make_sm("ModelB", guid="guid-b", skip_publish=False) workspace = MagicMock(spec=FabricWorkspace) workspace.workspace_id = "ws-123" workspace.endpoint = MagicMock() workspace.repository_items = {"SemanticModel": {"ModelA": model_a, "ModelB": model_b}} workspace.endpoint.invoke.return_value = { "body": {"value": [{"id": "old-conn", "connectivityType": "ShareableCloud", "connectionDetails": {}}]}, "status_code": 200, } connections = _make_connections("conn-001") connection_details = {"ModelA": "conn-001", "ModelB": "conn-001"} bind_semanticmodel_to_connection(workspace, connections, connection_details) called_urls = [c[1]["url"] for c in workspace.endpoint.invoke.call_args_list] assert any("guid-a" in url for url in called_urls), "ModelA should have been processed" assert any("guid-b" in url for url in called_urls), "ModelB should have been processed" # --------------------------------------------------------------------------- # Integration test through SemanticModelPublisher.post_publish_all() # --------------------------------------------------------------------------- def test_post_publish_all_skips_excluded_semantic_models(): """ End-to-end regression: SemanticModelPublisher.post_publish_all() must not attempt connection binding for models excluded by items_to_include. publish_all() marks excluded models with skip_publish=True; the guard in bind_semanticmodel_to_connection() must then prevent any API call for them. """ included_model = _make_sm("SalesModel", guid="sales-guid", skip_publish=False) excluded_model = _make_sm("DevModel", guid="", skip_publish=True) # set by publish_all() workspace = MagicMock(spec=FabricWorkspace) workspace.workspace_id = "ws-123" workspace.environment = "UAT" workspace.endpoint = MagicMock() workspace.repository_items = {"SemanticModel": {"SalesModel": included_model, "DevModel": excluded_model}} workspace.environment_parameter = { "semantic_model_binding": { "default": { "connection_id": {"_ALL_": "conn-001"} # applies to ALL models by default } } } workspace.items_to_include = ["SalesModel.SemanticModel"] # Simulate the connections API and bind call responses def fake_invoke(method, url, **_kwargs): if method == "GET" and url.split("?")[0].split("api.fabric.microsoft.com")[-1] == "/v1/connections": return { "body": { "value": [ { "id": "conn-001", "connectivityType": "ShareableCloud", "connectionDetails": {"type": "SQL", "path": "srv"}, } ] } } if method == "GET" and "/connections" in url: return { "body": { "value": [ { "id": "old-conn", "connectivityType": "ShareableCloud", "connectionDetails": {"type": "SQL", "path": "srv"}, } ] } } if method == "POST" and "bindConnection" in url: return {"status_code": 200} return {"body": {}} workspace.endpoint.invoke.side_effect = fake_invoke publisher = SemanticModelPublisher.__new__(SemanticModelPublisher) publisher.fabric_workspace_obj = workspace publisher.item_type = "SemanticModel" publisher.post_publish_all() called_urls = [c[1]["url"] for c in workspace.endpoint.invoke.call_args_list] # An empty GUID produces a path like "semanticModels//bindConnection"; strip the scheme to detect it. assert not any("//" in url.split("://", 1)[-1] for url in called_urls), f"Empty-GUID URL produced: {called_urls}" # DevModel (excluded, guid='') must never appear in any URL assert not any("DevModel" in url for url in called_urls), f"DevModel must be skipped entirely: {called_urls}" # SalesModel should be bound (bindConnection called with sales-guid) assert any("sales-guid" in url and "bindConnection" in url for url in called_urls), ( f"SalesModel bindConnection not called: {called_urls}" ) ================================================ FILE: tests/test_shortcut_exclude.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Test shortcut exclusion functionality.""" import json from unittest.mock import MagicMock, patch import pytest from fabric_cicd import constants from fabric_cicd._common._item import Item from fabric_cicd._items._lakehouse import LakehousePublisher, ShortcutPublisher from fabric_cicd.constants import FeatureFlag from fabric_cicd.fabric_workspace import FabricWorkspace @pytest.fixture def mock_fabric_workspace(): """Create a mock FabricWorkspace object.""" workspace = MagicMock(spec=FabricWorkspace) workspace.base_api_url = "https://api.fabric.microsoft.com/v1" workspace.shortcut_exclude_regex = None workspace.endpoint = MagicMock() # Mock the endpoint invoke method to return empty shortcuts list def mock_invoke(method, url, **_kwargs): if method == "GET" and "shortcuts" in url: return {"body": {"value": []}, "header": {}} if method == "POST" and "shortcuts" in url: return {"body": {"id": "mock-shortcut-id"}} return {"body": {}} workspace.endpoint.invoke.side_effect = mock_invoke # Mock parameter replacement methods to return content as-is workspace._replace_parameters = lambda file_obj, _item_obj: file_obj.contents workspace._replace_logical_ids = lambda contents: contents workspace._replace_workspace_ids = lambda contents: contents return workspace @pytest.fixture def mock_item(): """Create a mock Item object.""" item = MagicMock(spec=Item) item.name = "TestLakehouse" item.guid = "test-lakehouse-guid" return item def create_shortcut_file(shortcuts_data): """Helper to create a mock file object with shortcut data.""" file_obj = MagicMock() file_obj.name = "shortcuts.metadata.json" file_obj.contents = json.dumps(shortcuts_data) return file_obj def test_process_shortcuts_with_exclude_regex_filters_shortcuts(mock_fabric_workspace, mock_item): """Test that shortcut_exclude_regex correctly filters shortcuts from deployment.""" # Create shortcuts data shortcuts_data = [ { "name": "temp_shortcut1", "path": "/Tables", "target": { "type": "OneLake", "oneLake": { "path": "Tables/temp1", "itemId": "test-item-id", "workspaceId": "test-workspace-id", "artifactType": "Lakehouse", }, }, }, { "name": "production_shortcut", "path": "/Tables", "target": { "type": "OneLake", "oneLake": { "path": "Tables/prod", "itemId": "test-item-id", "workspaceId": "test-workspace-id", "artifactType": "Lakehouse", }, }, }, { "name": "temp_shortcut2", "path": "/Files", "target": { "type": "OneLake", "oneLake": { "path": "Files/temp2", "itemId": "test-item-id", "workspaceId": "test-workspace-id", "artifactType": "Lakehouse", }, }, }, ] # Create mock file with shortcuts shortcut_file = create_shortcut_file(shortcuts_data) mock_item.item_files = [shortcut_file] # Set exclude regex to filter out shortcuts starting with "temp_" mock_fabric_workspace.shortcut_exclude_regex = "^temp_.*" # Call process_shortcuts ShortcutPublisher(mock_fabric_workspace, mock_item).publish_all() # Verify that only the production_shortcut was published post_calls = [ call for call in mock_fabric_workspace.endpoint.invoke.call_args_list if call[1].get("method") == "POST" and "shortcuts" in call[1].get("url", "") ] # Should have only 1 shortcut published (production_shortcut) assert len(post_calls) == 1 # Verify the published shortcut is the production one published_shortcut = post_calls[0][1]["body"] assert published_shortcut["name"] == "production_shortcut" def test_process_shortcuts_without_exclude_regex_publishes_all(mock_fabric_workspace, mock_item): """Test that when shortcut_exclude_regex is None, all shortcuts are published.""" # Create shortcuts data shortcuts_data = [ { "name": "shortcut1", "path": "/Tables", "target": { "type": "OneLake", "oneLake": { "path": "Tables/s1", "itemId": "test-item-id", "workspaceId": "test-workspace-id", "artifactType": "Lakehouse", }, }, }, { "name": "shortcut2", "path": "/Files", "target": { "type": "OneLake", "oneLake": { "path": "Files/s2", "itemId": "test-item-id", "workspaceId": "test-workspace-id", "artifactType": "Lakehouse", }, }, }, ] # Create mock file with shortcuts shortcut_file = create_shortcut_file(shortcuts_data) mock_item.item_files = [shortcut_file] # No exclude regex set (None) mock_fabric_workspace.shortcut_exclude_regex = None # Call process_shortcuts ShortcutPublisher(mock_fabric_workspace, mock_item).publish_all() # Verify that both shortcuts were published post_calls = [ call for call in mock_fabric_workspace.endpoint.invoke.call_args_list if call[1].get("method") == "POST" and "shortcuts" in call[1].get("url", "") ] # Should have 2 shortcuts published assert len(post_calls) == 2 def test_process_shortcuts_exclude_regex_excludes_all_matching(mock_fabric_workspace, mock_item): """Test that shortcut_exclude_regex excludes all matching shortcuts.""" # Create shortcuts data with all matching the pattern shortcuts_data = [ { "name": "temp_shortcut1", "path": "/Tables", "target": { "type": "OneLake", "oneLake": { "path": "Tables/temp1", "itemId": "test-item-id", "workspaceId": "test-workspace-id", "artifactType": "Lakehouse", }, }, }, { "name": "temp_shortcut2", "path": "/Files", "target": { "type": "OneLake", "oneLake": { "path": "Files/temp2", "itemId": "test-item-id", "workspaceId": "test-workspace-id", "artifactType": "Lakehouse", }, }, }, ] # Create mock file with shortcuts shortcut_file = create_shortcut_file(shortcuts_data) mock_item.item_files = [shortcut_file] # Set exclude regex that matches all shortcuts mock_fabric_workspace.shortcut_exclude_regex = "^temp_.*" # Call process_shortcuts ShortcutPublisher(mock_fabric_workspace, mock_item).publish_all() # Verify that no shortcuts were published post_calls = [ call for call in mock_fabric_workspace.endpoint.invoke.call_args_list if call[1].get("method") == "POST" and "shortcuts" in call[1].get("url", "") ] # Should have 0 shortcuts published assert len(post_calls) == 0 def test_process_shortcuts_with_complex_regex_pattern(mock_fabric_workspace, mock_item): """Test shortcut exclusion with a more complex regex pattern.""" # Create shortcuts data shortcuts_data = [ { "name": "dev_temp_shortcut", "path": "/Tables", "target": { "type": "OneLake", "oneLake": { "path": "Tables/dev_temp", "itemId": "test-item-id", "workspaceId": "test-workspace-id", "artifactType": "Lakehouse", }, }, }, { "name": "prod_shortcut", "path": "/Tables", "target": { "type": "OneLake", "oneLake": { "path": "Tables/prod", "itemId": "test-item-id", "workspaceId": "test-workspace-id", "artifactType": "Lakehouse", }, }, }, { "name": "staging_temp_data", "path": "/Files", "target": { "type": "OneLake", "oneLake": { "path": "Files/staging_temp", "itemId": "test-item-id", "workspaceId": "test-workspace-id", "artifactType": "Lakehouse", }, }, }, ] # Create mock file with shortcuts shortcut_file = create_shortcut_file(shortcuts_data) mock_item.item_files = [shortcut_file] # Set exclude regex to filter shortcuts containing "_temp" mock_fabric_workspace.shortcut_exclude_regex = ".*_temp.*" # Call process_shortcuts ShortcutPublisher(mock_fabric_workspace, mock_item).publish_all() # Verify that only prod_shortcut was published post_calls = [ call for call in mock_fabric_workspace.endpoint.invoke.call_args_list if call[1].get("method") == "POST" and "shortcuts" in call[1].get("url", "") ] # Should have only 1 shortcut published (prod_shortcut) assert len(post_calls) == 1 # Verify the published shortcut is the prod one published_shortcut = post_calls[0][1]["body"] assert published_shortcut["name"] == "prod_shortcut" # ============================================================================= # Regression tests: items_to_include + shortcut publishing # ============================================================================= @pytest.fixture def shortcut_publish_enabled(): """Enable the ENABLE_SHORTCUT_PUBLISH feature flag for the duration of a test.""" original_flags = constants.FEATURE_FLAG.copy() constants.FEATURE_FLAG.add(FeatureFlag.ENABLE_SHORTCUT_PUBLISH.value) yield constants.FEATURE_FLAG.clear() constants.FEATURE_FLAG.update(original_flags) def _make_item(name: str, guid: str = "") -> Item: """Create a real Item for testing skip_publish and guid behaviour.""" return Item(type="Lakehouse", name=name, description="", guid=guid) def test_excluded_lakehouses_marked_skip_publish_with_items_to_include(): """ When items_to_include is set and the workspace contains lakehouses that are NOT in the include list, publish_all() must mark those lakehouses skip_publish=True so that post_publish_all() does not attempt to publish their shortcuts (which would fail with a 400 error because guid is ""). """ lh_bronze = _make_item("lh_bronze", guid="bronze-guid") lh_silver = _make_item("lh_silver", guid="") # not deployed to this environment workspace = MagicMock(spec=FabricWorkspace) workspace.repository_items = {"Lakehouse": {"lh_bronze": lh_bronze, "lh_silver": lh_silver}} workspace.items_to_include = ["lh_bronze.Lakehouse"] publisher = LakehousePublisher.__new__(LakehousePublisher) publisher.fabric_workspace_obj = workspace # items_to_include filtering inside get_items_to_publish() items = publisher.get_items_to_publish() assert "lh_bronze" in items assert "lh_silver" not in items # Simulate what publish_all() now does: mark excluded items skip_publish=True all_items = workspace.repository_items.get("Lakehouse", {}) for item_name, item_obj in all_items.items(): if item_name not in items: item_obj.skip_publish = True # The excluded lakehouse must be marked as skip_publish assert lh_silver.skip_publish is True # The included lakehouse must NOT be marked as skip_publish assert lh_bronze.skip_publish is False @pytest.mark.usefixtures("shortcut_publish_enabled") def test_lakehouses_without_guid_are_not_shortcut_published(): """ Regression test for the guid guard in post_publish_all(). Even if skip_publish is somehow False for a lakehouse with no guid, the guid guard in post_publish_all() must prevent shortcut publishing for it, avoiding the 'items//shortcuts' URL that returns a 400 error. """ lh_bronze = _make_item("lh_bronze", guid="bronze-guid") lh_silver = _make_item("lh_silver", guid="") # empty guid, skip_publish stays False workspace = MagicMock(spec=FabricWorkspace) workspace.repository_items = {"Lakehouse": {"lh_bronze": lh_bronze, "lh_silver": lh_silver}} workspace.items_to_include = ["lh_bronze.Lakehouse"] workspace.shortcut_exclude_regex = None publisher = LakehousePublisher.__new__(LakehousePublisher) publisher.fabric_workspace_obj = workspace with patch("fabric_cicd._items._lakehouse.ShortcutPublisher") as mock_shortcut_cls: publisher.post_publish_all() # ShortcutPublisher should only be instantiated for lh_bronze (has guid) calls = mock_shortcut_cls.call_args_list assert len(calls) == 1 assert calls[0][0][1] is lh_bronze @pytest.mark.usefixtures("shortcut_publish_enabled") def test_publish_all_marks_excluded_items_skip_publish(): """ End-to-end regression: publish_all() must mark items excluded by items_to_include as skip_publish=True before post_publish_all() runs, so that shortcut publishing is never attempted for lakehouses with empty guids. """ lh_bronze = _make_item("lh_bronze", guid="bronze-guid") lh_silver = _make_item("lh_silver", guid="") # not deployed in this environment workspace = MagicMock(spec=FabricWorkspace) workspace.repository_items = {"Lakehouse": {"lh_bronze": lh_bronze, "lh_silver": lh_silver}} workspace.items_to_include = ["lh_bronze.Lakehouse"] workspace.shortcut_exclude_regex = None publisher = LakehousePublisher.__new__(LakehousePublisher) publisher.fabric_workspace_obj = workspace publisher.item_type = "Lakehouse" # Track which items get shortcut-published shortcut_published_guids = [] def fake_shortcut_publish_all(self_inner): shortcut_published_guids.append(self_inner.item_obj.guid) with ( patch.object(LakehousePublisher, "pre_publish_all", return_value=None), patch.object(LakehousePublisher, "publish_one", return_value=None), patch("fabric_cicd._items._lakehouse.ShortcutPublisher.publish_all", fake_shortcut_publish_all), ): publisher.publish_all() # lh_silver must be marked as skip_publish after publish_all() assert lh_silver.skip_publish is True # lh_bronze was in items_to_include so must NOT be skip_publish assert lh_bronze.skip_publish is False # Shortcuts must only be published for lh_bronze (has guid); never for lh_silver assert shortcut_published_guids == ["bronze-guid"] ================================================ FILE: tests/test_subfolders.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """Test subfolder creation and modification in the fabric workspace.""" import json import os import re from unittest.mock import MagicMock, patch import pytest from fixtures.credentials import DummyTokenCredential from fabric_cicd.fabric_workspace import FabricWorkspace @pytest.fixture def mock_endpoint(): """Mock FabricEndpoint to avoid real API calls.""" mock = MagicMock() mock.invoke.return_value = {"body": {"value": []}, "header": {}} return mock @pytest.fixture def temp_workspace_dir(tmp_path): """Create a temporary directory structure for testing.""" # Use pytest's tmp_path for better isolation return tmp_path @pytest.fixture def valid_workspace_id(): """Return a valid workspace ID in GUID format.""" return "12345678-1234-5678-abcd-1234567890ab" def create_platform_file(item_path, item_type="Notebook", item_name="Test Item"): """Create a .platform file for an item.""" platform_file_path = item_path / ".platform" item_path.mkdir(parents=True, exist_ok=True) metadata_content = { "metadata": { "type": item_type, "displayName": item_name, "description": f"Test {item_type}", }, "config": {"logicalId": f"test-logical-id-{item_name}"}, } with platform_file_path.open("w", encoding="utf-8") as f: json.dump(metadata_content, f, ensure_ascii=False) # Create a dummy content file with (item_path / "dummy.txt").open("w", encoding="utf-8") as f: f.write("Dummy file") return metadata_content @pytest.fixture def repository_with_subfolders(tmp_path): """Create a repository with subfolders for testing - isolated per test.""" # Create root level items create_platform_file(tmp_path / "RootNotebook.Notebook", item_type="Notebook", item_name="Root Notebook") create_platform_file(tmp_path / "RootPipeline.DataPipeline", item_type="DataPipeline", item_name="Root Pipeline") # Create first level subfolders with items create_platform_file( tmp_path / "Folder1" / "Folder1Notebook.Notebook", item_type="Notebook", item_name="Folder1 Notebook" ) create_platform_file( tmp_path / "Folder2" / "Folder2Pipeline.DataPipeline", item_type="DataPipeline", item_name="Folder2 Pipeline" ) # Create second level subfolders with items create_platform_file( tmp_path / "Folder1" / "Subfolder1" / "Subfolder1Notebook.Notebook", item_type="Notebook", item_name="Subfolder1 Notebook", ) create_platform_file( tmp_path / "Folder2" / "Subfolder2" / "Subfolder2Pipeline.DataPipeline", item_type="DataPipeline", item_name="Subfolder2 Pipeline", ) # Create empty folder (should not be included in repository_folders) (tmp_path / "EmptyFolder").mkdir(parents=True, exist_ok=True) # Create a folder with only empty subfolders (should not be included) (tmp_path / "FolderWithEmptySubfolders" / "EmptySubfolder").mkdir(parents=True, exist_ok=True) return tmp_path @pytest.fixture def patched_fabric_workspace(mock_endpoint): """Return a factory function to create a patched FabricWorkspace.""" def _create_workspace(workspace_id, repository_directory, item_type_in_scope, **kwargs): fabric_endpoint_patch = patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint) parameter_patch = patch.object( FabricWorkspace, "_refresh_parameter_file", new=lambda self: setattr(self, "environment_parameter", {}) ) with fabric_endpoint_patch, parameter_patch: return FabricWorkspace( workspace_id=workspace_id, repository_directory=repository_directory, item_type_in_scope=item_type_in_scope, token_credential=DummyTokenCredential(), **kwargs, ) return _create_workspace def test_refresh_repository_folders(repository_with_subfolders, patched_fabric_workspace, valid_workspace_id): """Test the _refresh_repository_folders method.""" workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(repository_with_subfolders), item_type_in_scope=["Notebook", "DataPipeline"], ) # Call the method under test workspace._refresh_repository_folders() # Verify folders are correctly identified assert "/Folder1" in workspace.repository_folders assert "/Folder2" in workspace.repository_folders assert "/Folder1/Subfolder1" in workspace.repository_folders assert "/Folder2/Subfolder2" in workspace.repository_folders # Verify empty folders are not included assert "/EmptyFolder" not in workspace.repository_folders assert "/FolderWithEmptySubfolders" not in workspace.repository_folders assert "/FolderWithEmptySubfolders/EmptySubfolder" not in workspace.repository_folders # Verify all folder IDs are initially empty strings for folder_id in workspace.repository_folders.values(): assert folder_id == "" def test_publish_folders_hierarchy(repository_with_subfolders, patched_fabric_workspace, valid_workspace_id): """Test that the folder hierarchy is correctly established.""" workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(repository_with_subfolders), item_type_in_scope=["Notebook", "DataPipeline"], ) # Call the method under test workspace._refresh_repository_folders() # Verify folders are correctly identified assert "/Folder1" in workspace.repository_folders assert "/Folder2" in workspace.repository_folders assert "/Folder1/Subfolder1" in workspace.repository_folders assert "/Folder2/Subfolder2" in workspace.repository_folders # Sort folders by path depth sorted_folders = sorted(workspace.repository_folders.keys(), key=lambda path: path.count("/")) # Check parent-child relationships in the sorted folder list # Parents should always come before their children assert sorted_folders.index("/Folder1") < sorted_folders.index("/Folder1/Subfolder1") assert sorted_folders.index("/Folder2") < sorted_folders.index("/Folder2/Subfolder2") # Verify direct parent-child relationships by checking path structure for folder_path in workspace.repository_folders: if folder_path.count("/") > 1: # It's a subfolder parent_path = "/".join(folder_path.split("/")[:-1]) assert parent_path in workspace.repository_folders, ( f"Parent folder {parent_path} not found for {folder_path}" ) def test_folder_hierarchy_preservation(repository_with_subfolders, patched_fabric_workspace, valid_workspace_id): """Test that the folder hierarchy is preserved when reusing existing folders.""" workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(repository_with_subfolders), item_type_in_scope=["Notebook", "DataPipeline"], ) # Mock the deployed folders API to return existing folders folder1_id = "folder1-id-12345" folder2_id = "folder2-id-67890" def mock_invoke_side_effect(method, url, **_kwargs): if method == "GET" and url.endswith("/folders"): # Mock API response for existing folders return { "body": { "value": [ {"id": folder1_id, "displayName": "Folder1", "parentFolderId": None}, {"id": folder2_id, "displayName": "Folder2", "parentFolderId": None}, ] }, "header": {}, } return {"body": {"value": []}, "header": {}} workspace.endpoint.invoke.side_effect = mock_invoke_side_effect # Call methods in the intended order workspace._refresh_repository_folders() workspace._refresh_deployed_folders() # Capture initial repository folders initial_folders = set(workspace.repository_folders.keys()) # Verify the folder hierarchy remains intact assert set(workspace.repository_folders.keys()) == initial_folders # Verify deployed folder IDs were detected correctly assert workspace.deployed_folders["/Folder1"] == folder1_id assert workspace.deployed_folders["/Folder2"] == folder2_id # Verify subfolder paths still exist in repository (even if not deployed) assert "/Folder1/Subfolder1" in workspace.repository_folders assert "/Folder2/Subfolder2" in workspace.repository_folders def test_item_folder_association(repository_with_subfolders, valid_workspace_id): """Test that items are correctly associated with their parent folders.""" # Set up mock folder IDs folder1_id = "folder1-id-12345" folder2_id = "folder2-id-67890" subfolder1_id = "subfolder1-id-12345" subfolder2_id = "subfolder2-id-67890" # Mock responses for API calls def mock_invoke_side_effect(method, url, **_kwargs): if method == "GET" and url.endswith("/items"): return {"body": {"value": []}} if method == "GET" and url.endswith("/folders"): # Mock API response for deployed folders return { "body": { "value": [ {"id": folder1_id, "displayName": "Folder1", "parentFolderId": None}, {"id": folder2_id, "displayName": "Folder2", "parentFolderId": None}, {"id": subfolder1_id, "displayName": "Subfolder1", "parentFolderId": folder1_id}, {"id": subfolder2_id, "displayName": "Subfolder2", "parentFolderId": folder2_id}, ] }, "header": {}, } return {"body": {"value": []}} mock_endpoint = MagicMock() mock_endpoint.invoke.side_effect = mock_invoke_side_effect fabric_endpoint_patch = patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint) parameter_patch = patch.object( FabricWorkspace, "_refresh_parameter_file", new=lambda self: setattr(self, "environment_parameter", {}) ) with fabric_endpoint_patch, parameter_patch: workspace = FabricWorkspace( workspace_id=valid_workspace_id, repository_directory=str(repository_with_subfolders), item_type_in_scope=["Notebook", "DataPipeline"], token_credential=DummyTokenCredential(), ) # Call methods in the intended order to populate folder structures workspace._refresh_repository_folders() workspace._refresh_deployed_folders() # Simulate the effect of _publish_folders by updating repository_folders # with deployed folder IDs (this normally happens in _publish_folders) for folder_path, folder_id in workspace.deployed_folders.items(): if folder_path in workspace.repository_folders: workspace.repository_folders[folder_path] = folder_id workspace._refresh_repository_items() # Verify folder IDs are correctly assigned to items assert workspace.repository_items["Notebook"]["Root Notebook"].folder_id == "" assert workspace.repository_items["Notebook"]["Folder1 Notebook"].folder_id == folder1_id assert workspace.repository_items["Notebook"]["Subfolder1 Notebook"].folder_id == subfolder1_id assert workspace.repository_items["DataPipeline"]["Root Pipeline"].folder_id == "" assert workspace.repository_items["DataPipeline"]["Folder2 Pipeline"].folder_id == folder2_id assert workspace.repository_items["DataPipeline"]["Subfolder2 Pipeline"].folder_id == subfolder2_id def test_deeply_nested_subfolders(tmp_path, patched_fabric_workspace, valid_workspace_id): """Test handling of deeply nested folder structures (15+ levels deep).""" # Create a deeply nested folder structure current_path = tmp_path folder_names = [] # Create 15 levels of nested folders (adjust depth for Windows if needed) depth = 15 prefix = "Level" if os.name == "nt": depth = 8 prefix = "L" for i in range(1, depth + 1): folder_name = f"{prefix}{i:02d}" folder_names.append(folder_name) current_path = current_path / folder_name # Create an item in the deepest folder create_platform_file( current_path / "DeepNotebook.Notebook", item_type="Notebook", item_name="Deep Notebook", ) mid_level_path = tmp_path for i in range(min(7, depth)): # Create item at level 7 mid_level_path = mid_level_path / folder_names[i] create_platform_file( mid_level_path / "MidLevelNotebook.Notebook", item_type="Notebook", item_name="Mid Level Notebook", ) workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(tmp_path), item_type_in_scope=["Notebook"], ) # Test that _refresh_repository_folders can handle deep nesting workspace._refresh_repository_folders() # Verify all folder levels were detected expected_deep_path = "/" + "/".join(folder_names) expected_mid_path = "/" + "/".join(folder_names[: min(7, depth)]) assert expected_deep_path in workspace.repository_folders assert expected_mid_path in workspace.repository_folders # Verify folder hierarchy ordering (parents before children) sorted_folders = sorted(workspace.repository_folders.keys(), key=lambda path: path.count("/")) for i in range(1, depth): current_level_path = "/" + "/".join(folder_names[:i]) next_level_path = "/" + "/".join(folder_names[: i + 1]) if current_level_path in workspace.repository_folders and next_level_path in workspace.repository_folders: assert sorted_folders.index(current_level_path) < sorted_folders.index(next_level_path) # Verify no stack overflow or performance issues by checking reasonable execution time import time start_time = time.time() workspace._refresh_repository_folders() execution_time = time.time() - start_time # Should complete in reasonable time (< 1 second for 15 levels) assert execution_time < 1.0, f"Deep folder processing took too long: {execution_time:.2f}s" def test_folder_rename_operations(tmp_path, patched_fabric_workspace, valid_workspace_id): """Test folder rename operations and verify child items and subfolders are updated correctly.""" # Create initial folder structure in isolated tmp_path original_folder = tmp_path / "OriginalFolder" original_subfolder = original_folder / "OriginalSubfolder" # Create items in original folders create_platform_file(original_folder / "ParentNotebook.Notebook", item_type="Notebook", item_name="Parent Notebook") create_platform_file( original_subfolder / "ChildNotebook.Notebook", item_type="Notebook", item_name="Child Notebook" ) workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(tmp_path), item_type_in_scope=["Notebook"], ) # Initial state workspace._refresh_repository_folders() workspace._refresh_repository_items() assert "/OriginalFolder" in workspace.repository_folders assert "/OriginalFolder/OriginalSubfolder" in workspace.repository_folders # Create a separate workspace instance for testing renamed structure # to avoid contaminating the original workspace state renamed_tmp_path = tmp_path.parent / "renamed_workspace" renamed_tmp_path.mkdir() # Create the renamed folder structure in the new location renamed_folder = renamed_tmp_path / "RenamedFolder" renamed_subfolder = renamed_folder / "RenamedSubfolder" # Create items in renamed folders create_platform_file(renamed_folder / "ParentNotebook.Notebook", item_type="Notebook", item_name="Parent Notebook") create_platform_file(renamed_subfolder / "ChildNotebook.Notebook", item_type="Notebook", item_name="Child Notebook") # Create new workspace instance for renamed structure renamed_workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(renamed_tmp_path), item_type_in_scope=["Notebook"], ) # Refresh after "rename" (using the new workspace) renamed_workspace._refresh_repository_folders() renamed_workspace._refresh_repository_items() # Verify old folder paths are no longer present in renamed workspace assert "/OriginalFolder" not in renamed_workspace.repository_folders assert "/OriginalFolder/OriginalSubfolder" not in renamed_workspace.repository_folders # Verify new folder paths are detected in renamed workspace assert "/RenamedFolder" in renamed_workspace.repository_folders assert "/RenamedFolder/RenamedSubfolder" in renamed_workspace.repository_folders # Verify items are detected in new locations assert "Parent Notebook" in renamed_workspace.repository_items["Notebook"] assert "Child Notebook" in renamed_workspace.repository_items["Notebook"] # Verify folder hierarchy is maintained sorted_folders = sorted(renamed_workspace.repository_folders.keys(), key=lambda path: path.count("/")) assert sorted_folders.index("/RenamedFolder") < sorted_folders.index("/RenamedFolder/RenamedSubfolder") def test_special_character_handling(tmp_path, patched_fabric_workspace, valid_workspace_id): """Test handling of special characters in folder names.""" test_cases = [ # Valid cases - should be accepted ("ValidFolder", True, "Basic valid folder name"), ("Folder_With_Underscores", True, "Underscores should be valid"), ("Folder-With-Hyphens", True, "Hyphens should be valid"), ("Folder With Spaces", True, "Spaces should be valid"), ("FolderWithUnicode_测试", True, "Unicode characters should be valid"), ("FolderWith123Numbers", True, "Numbers should be valid"), (" SpacesAroundName ", True, "Leading/trailing spaces should be handled"), # Invalid cases - should be rejected by regex ("Folder*WithAsterisk", False, "Asterisk should be invalid"), ("Folder#WithHash", False, "Hash should be invalid"), ("FolderWithBracket", False, "Angle bracket should be invalid"), ("Folder:WithColon", False, "Colon should be invalid"), ('Folder"WithQuote', False, "Quote should be invalid"), ("Folder|WithPipe", False, "Pipe should be invalid"), ("Folder?WithQuestion", False, "Question mark should be invalid"), ("Folder\\WithBackslash", False, "Backslash should be invalid"), ("Folder/WithSlash", False, "Forward slash should be invalid"), ("Folder{WithBrace", False, "Curly brace should be invalid"), ("Folder}WithBrace", False, "Curly brace should be invalid"), ("Folder~WithTilde", False, "Tilde should be invalid"), ("Folder.WithDot", False, "Dot should be invalid"), ("Folder%WithPercent", False, "Percent should be invalid"), ("Folder&WithAmpersand", False, "Ampersand should be invalid"), ] from fabric_cicd import constants # Test regex validation for each case for folder_name, should_be_valid, description in test_cases: has_invalid_chars = bool(re.search(constants.INVALID_FOLDER_CHAR_REGEX, folder_name)) if should_be_valid: assert not has_invalid_chars, f"{description}: '{folder_name}' should be valid but was rejected" else: assert has_invalid_chars, f"{description}: '{folder_name}' should be invalid but was accepted" # Test actual folder creation with some valid cases valid_folders = ["ValidFolder", "Folder_With_Underscores", "Folder-With-Hyphens", "FolderWithUnicode_测试"] for folder_name in valid_folders: folder_path = tmp_path / folder_name create_platform_file( folder_path / "TestNotebook.Notebook", item_type="Notebook", item_name=f"Test {folder_name}" ) workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(tmp_path), item_type_in_scope=["Notebook"], ) workspace._refresh_repository_folders() # Verify valid folders were detected for folder_name in valid_folders: expected_path = f"/{folder_name}" assert expected_path in workspace.repository_folders, f"Valid folder '{folder_name}' was not detected" def test_parent_folder_with_only_subfolder_containing_items(tmp_path, patched_fabric_workspace, valid_workspace_id): """Test scenario where parent folder contains only a subfolder with items (no direct items in parent).""" # Create parent folder with NO direct items parent_folder = tmp_path / "ParentWithNoDirectItems" parent_folder.mkdir(parents=True, exist_ok=True) # Create subfolder with items (parent has no direct items) create_platform_file( parent_folder / "SubfolderWithItems" / "SubfolderNotebook.Notebook", item_type="Notebook", item_name="Subfolder Notebook", ) # Also create a more complex scenario with nested empty parents deeply_nested_parent = tmp_path / "Level1EmptyParent" / "Level2EmptyParent" deeply_nested_parent.mkdir(parents=True, exist_ok=True) create_platform_file( deeply_nested_parent / "FinalSubfolder" / "DeepNestedNotebook.Notebook", item_type="Notebook", item_name="Deep Nested Notebook", ) workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(tmp_path), item_type_in_scope=["Notebook"], ) workspace._refresh_repository_folders() workspace._refresh_repository_items() # Verify parent folders are detected even though they have no direct items assert "/ParentWithNoDirectItems" in workspace.repository_folders assert "/ParentWithNoDirectItems/SubfolderWithItems" in workspace.repository_folders # Verify deeply nested scenario assert "/Level1EmptyParent" in workspace.repository_folders assert "/Level1EmptyParent/Level2EmptyParent" in workspace.repository_folders assert "/Level1EmptyParent/Level2EmptyParent/FinalSubfolder" in workspace.repository_folders # Verify folder hierarchy is maintained (parents before children) sorted_folders = sorted(workspace.repository_folders.keys(), key=lambda path: path.count("/")) parent_idx = sorted_folders.index("/ParentWithNoDirectItems") subfolder_idx = sorted_folders.index("/ParentWithNoDirectItems/SubfolderWithItems") assert parent_idx < subfolder_idx, "Parent folder should come before subfolder" # Verify items are correctly associated with their folders assert "Subfolder Notebook" in workspace.repository_items["Notebook"] assert "Deep Nested Notebook" in workspace.repository_items["Notebook"] # Verify that parent folders have empty folder IDs (since they have no direct deployment) # but their subfolders would get proper folder IDs when deployed assert workspace.repository_folders["/ParentWithNoDirectItems"] == "" assert workspace.repository_folders["/Level1EmptyParent"] == "" assert workspace.repository_folders["/Level1EmptyParent/Level2EmptyParent"] == "" def test_large_number_of_folders_and_items(tmp_path, patched_fabric_workspace, valid_workspace_id): """Test performance and scalability with a large number of folders and items.""" import time # Create a large number of folders and items (100 folders with multiple items each) num_folders = 100 items_per_folder = 3 # Create folders at multiple levels for i in range(num_folders): if i < 50: # First 50 folders at root level folder_path = tmp_path / f"Folder{i:03d}" else: # Next 50 folders nested under first 25 folders parent_idx = (i - 50) % 25 folder_path = tmp_path / f"Folder{parent_idx:03d}" / f"Subfolder{i:03d}" # Create multiple items in each folder for j in range(items_per_folder): create_platform_file( folder_path / f"Item{j:02d}.Notebook", item_type="Notebook", item_name=f"Item {j:02d} in Folder {i:03d}" ) workspace = patched_fabric_workspace( workspace_id=valid_workspace_id, repository_directory=str(tmp_path), item_type_in_scope=["Notebook"], ) # Test _refresh_repository_folders performance start_time = time.time() workspace._refresh_repository_folders() folders_time = time.time() - start_time # Verify we detected a reasonable number of folders assert len(workspace.repository_folders) >= 50, "Should detect at least 50 folders" assert len(workspace.repository_folders) <= 125, "Should not detect more than 125 folders" # Test _refresh_repository_items performance start_time = time.time() workspace._refresh_repository_items() items_time = time.time() - start_time # Verify we detected the expected number of items expected_items = num_folders * items_per_folder assert len(workspace.repository_items["Notebook"]) == expected_items # Performance assertions - should complete in reasonable time assert folders_time < 15.0, f"Folder detection took too long: {folders_time:.2f}s" assert items_time < 30.0, f"Item detection took too long: {items_time:.2f}s" # Memory usage test - verify folder hierarchy is correct # Check that parent-child relationships are maintained even with large numbers nested_folders = [path for path in workspace.repository_folders if path.count("/") > 1] for folder_path in nested_folders: parent_path = "/".join(folder_path.split("/")[:-1]) assert parent_path in workspace.repository_folders, f"Parent {parent_path} not found for {folder_path}" # Test that folder sorting still works correctly with large numbers sorted_folders = sorted(workspace.repository_folders.keys(), key=lambda path: path.count("/")) # Verify sorting is correct - all level 1 folders should come before level 2 folders level_1_folders = [f for f in sorted_folders if f.count("/") == 1] level_2_folders = [f for f in sorted_folders if f.count("/") == 2] if level_1_folders and level_2_folders: last_level_1_index = max(sorted_folders.index(f) for f in level_1_folders) first_level_2_index = min(sorted_folders.index(f) for f in level_2_folders) assert last_level_1_index < first_level_2_index, "Folder sorting is incorrect with large numbers" ================================================ FILE: tests/test_validate_env_vars.py ================================================ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. import os import pytest from fabric_cicd._common._exceptions import InputError from fabric_cicd._common._validate_env_vars import ( _VALID_HOSTNAME_REGEX, validate_api_url, validate_env_var_api_url, ) class TestValidHostnameRegex: """Tests for the VALID_HOSTNAME_REGEX pattern.""" @pytest.mark.parametrize( "hostname", [ "api.fabric.microsoft.com", "api.powerbi.com", "myapi.fabric.microsoft.com", "myapi.powerbi.com", "someapi.fabric.microsoft.com", "someapi.powerbi.com", "some.api.fabric.microsoft.com", "some.api.powerbi.com", "my-org.api.fabric.microsoft.com", "a.b.api.fabric.microsoft.com", "abcdef01234567890abcdef012345678.z01.w.api.fabric.microsoft.com", "abcdef01234567890abcdef012345678.z42.w.api.powerbi.com", # Hyphen and underscore in labels "staging-api.fabric.microsoft.com", "staging-api.powerbi.com", "my_org.api.fabric.microsoft.com", # Deeply nested subdomains "a.b.c.d.api.fabric.microsoft.com", # Numeric subdomain label "123.api.powerbi.com", # Case insensitive "API.fabric.microsoft.com", "api.FABRIC.microsoft.com", "api.fabric.MICROSOFT.com", "Api.PowerBI.Com", ], ) def test_valid_hostnames(self, hostname): assert _VALID_HOSTNAME_REGEX.match(hostname), f"Expected '{hostname}' to be valid" @pytest.mark.parametrize( "hostname", [ "fabric.microsoft.com", "powerbi.com", "contoso.com", "api.fabric.microsoft.com.contoso.com", "api.powerbi.com.contoso.com", "", "https://api.fabric.microsoft.com", "api.fabric.microsoft.com/path", "random.hostname.com", # Label doesn't end with 'api' "dfs.fabric.microsoft.com", "management.fabric.microsoft.com", "apix.fabric.microsoft.com", "my-apix.fabric.microsoft.com", # Trailing dot "api.fabric.microsoft.com.", # Leading dot ".api.fabric.microsoft.com", # Domain suffix spoofing "api.fabric.microsoft.com.br", "api.fabric.microsoft.community", "api.notfabric.microsoft.com", "api.fakepowerbi.com", ], ) def test_invalid_hostnames(self, hostname): assert not _VALID_HOSTNAME_REGEX.match(hostname), f"Expected '{hostname}' to be invalid" class TestValidateApiUrl: """Tests for the standalone validate_api_url function.""" def test_accepts_valid_fabric_url(self): result = validate_api_url("https://api.fabric.microsoft.com", "test_label") assert result == "https://api.fabric.microsoft.com" def test_accepts_valid_powerbi_url(self): result = validate_api_url("https://api.powerbi.com", "test_label") assert result == "https://api.powerbi.com" def test_strips_trailing_slash(self): result = validate_api_url("https://api.fabric.microsoft.com/", "test_label") assert result == "https://api.fabric.microsoft.com" def test_label_appears_in_error_message(self): with pytest.raises(InputError, match="my_config_key"): validate_api_url("http://api.fabric.microsoft.com", "my_config_key") def test_rejects_empty_string(self): with pytest.raises(InputError, match="must resolve to a non-empty string"): validate_api_url("", "test_label") def test_rejects_whitespace_only(self): with pytest.raises(InputError, match="must resolve to a non-empty string"): validate_api_url(" ", "test_label") def test_rejects_http_scheme(self): with pytest.raises(InputError, match="HTTPS scheme"): validate_api_url("http://api.fabric.microsoft.com", "test_label") def test_rejects_invalid_hostname(self): with pytest.raises(InputError, match="invalid hostname"): validate_api_url("https://evil.com", "test_label") def test_rejects_path_components(self): with pytest.raises(InputError, match="root URL without path components"): validate_api_url("https://api.fabric.microsoft.com/v1/workspaces", "test_label") def test_accepts_private_link_url(self): url = "https://abcdef01234567890abcdef012345678.z01.w.api.fabric.microsoft.com" result = validate_api_url(url, "test_label") assert result == url class TestValidateApiUrlHostname: """Tests for the validate_env_var_api_url function (env var wrapper).""" def test_returns_default_when_env_not_set(self): env_var = "TEST_HOSTNAME_NOT_SET_12345" assert env_var not in os.environ result = validate_env_var_api_url(env_var, "https://api.fabric.microsoft.com") assert result == "https://api.fabric.microsoft.com" def test_returns_default_powerbi_url_when_env_not_set(self): env_var = "TEST_HOSTNAME_NOT_SET_12345" assert env_var not in os.environ result = validate_env_var_api_url(env_var, "https://api.powerbi.com") assert result == "https://api.powerbi.com" def test_returns_env_value_when_set(self, monkeypatch): monkeypatch.setenv("TEST_HOSTNAME_VAR", "https://api.powerbi.com") result = validate_env_var_api_url("TEST_HOSTNAME_VAR", "https://api.fabric.microsoft.com") assert result == "https://api.powerbi.com" def test_rejects_path_components(self, monkeypatch): monkeypatch.setenv("TEST_HOSTNAME_VAR", "https://api.fabric.microsoft.com/v1/workspaces") with pytest.raises(InputError, match="root URL without path components"): validate_env_var_api_url("TEST_HOSTNAME_VAR", "https://api.fabric.microsoft.com") def test_raises_on_invalid_hostname(self, monkeypatch): monkeypatch.setenv("TEST_HOSTNAME_VAR", "https://evil.com") with pytest.raises(InputError, match="invalid hostname"): validate_env_var_api_url("TEST_HOSTNAME_VAR", "https://api.fabric.microsoft.com") def test_raises_on_empty_hostname(self, monkeypatch): monkeypatch.setenv("TEST_HOSTNAME_VAR", "") with pytest.raises(InputError, match="must resolve to a non-empty string"): validate_env_var_api_url("TEST_HOSTNAME_VAR", "https://api.fabric.microsoft.com") def test_prefixed_hostname_accepted(self, monkeypatch): monkeypatch.setenv("TEST_HOSTNAME_VAR", "https://myapi.fabric.microsoft.com") result = validate_env_var_api_url("TEST_HOSTNAME_VAR", "https://api.fabric.microsoft.com") assert result == "https://myapi.fabric.microsoft.com" def test_workspace_id_pattern_accepted(self, monkeypatch): url = "https://abcdef01234567890abcdef012345678.z01.w.api.fabric.microsoft.com" monkeypatch.setenv("TEST_HOSTNAME_VAR", url) result = validate_env_var_api_url("TEST_HOSTNAME_VAR", "https://api.fabric.microsoft.com") assert result == url def test_dotted_prefix_hostname_accepted(self, monkeypatch): monkeypatch.setenv("TEST_HOSTNAME_VAR", "https://some.api.fabric.microsoft.com") result = validate_env_var_api_url("TEST_HOSTNAME_VAR", "https://api.fabric.microsoft.com") assert result == "https://some.api.fabric.microsoft.com" def test_hostname_without_scheme_rejected(self, monkeypatch): monkeypatch.setenv("TEST_HOSTNAME_VAR", "api.fabric.microsoft.com") with pytest.raises(InputError, match="Invalid or missing scheme"): validate_env_var_api_url("TEST_HOSTNAME_VAR", "https://api.fabric.microsoft.com") def test_rejects_http_scheme(self, monkeypatch): monkeypatch.setenv("TEST_HOSTNAME_VAR", "http://api.fabric.microsoft.com") with pytest.raises(InputError, match="Invalid or missing scheme"): validate_env_var_api_url("TEST_HOSTNAME_VAR", "https://api.fabric.microsoft.com") def test_rejects_ftp_scheme(self, monkeypatch): monkeypatch.setenv("TEST_HOSTNAME_VAR", "ftp://api.fabric.microsoft.com") with pytest.raises(InputError, match="Invalid or missing scheme"): validate_env_var_api_url("TEST_HOSTNAME_VAR", "https://api.fabric.microsoft.com") def test_raises_on_whitespace_only(self, monkeypatch): monkeypatch.setenv("TEST_HOSTNAME_VAR", " ") with pytest.raises(InputError, match="must resolve to a non-empty string"): validate_env_var_api_url("TEST_HOSTNAME_VAR", "https://api.fabric.microsoft.com") def test_strips_trailing_slash(self, monkeypatch): monkeypatch.setenv("TEST_HOSTNAME_VAR", "https://api.fabric.microsoft.com/") result = validate_env_var_api_url("TEST_HOSTNAME_VAR", "https://api.fabric.microsoft.com") assert result == "https://api.fabric.microsoft.com" def test_rejects_url_with_no_authority(self, monkeypatch): monkeypatch.setenv("TEST_HOSTNAME_VAR", "https:///") with pytest.raises(InputError, match="invalid hostname"): validate_env_var_api_url("TEST_HOSTNAME_VAR", "https://api.fabric.microsoft.com")