Showing preview only (2,820K chars total). Download the full file or copy to clipboard to get everything.
Repository: mrpaulandrew/procfwk
Branch: master
Commit: f51cb458ef7e
Files: 344
Total size: 2.6 MB
Directory structure:
gitextract_8_ebrzfj/
├── .gitattributes
├── .github/
│ ├── FUNDING.yml
│ └── ISSUE_TEMPLATE/
│ ├── bug-found.md
│ ├── feature-request.md
│ └── help---support-request.md
├── .gitignore
├── .vscode/
│ ├── extensions.json
│ ├── launch.json
│ ├── settings.json
│ └── tasks.json
├── ARM Templates/
│ ├── Data Factory/
│ │ ├── v1.0 Export.json
│ │ ├── v1.1 Export.json
│ │ ├── v1.2 Export.json
│ │ ├── v1.3 Export.json
│ │ ├── v1.4 Export.json
│ │ ├── v1.5 Export.json
│ │ ├── v1.6 Export.json
│ │ ├── v1.7 Export.json
│ │ ├── v1.8 Export.json
│ │ ├── v1.8.3 Export.json
│ │ ├── v1.8.5 Export.json
│ │ ├── v1.8.6 Export.json
│ │ ├── v1.9 Export.json
│ │ ├── v1.9.1 Export.json
│ │ ├── v1.9.2 Export.json
│ │ └── v2.0 Export.json
│ ├── Functions App/
│ │ └── v1.6 Export.json
│ ├── SQL Database/
│ │ └── v1.6 Export.json
│ └── Synapse/
│ └── v2.0 Export.json
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── DataFactory/
│ ├── dataset/
│ │ └── GetSetMetadata.json
│ ├── factory/
│ │ └── FrameworkFactory.json
│ ├── integrationRuntime/
│ │ └── AzureIR-UKSouth.json
│ ├── linkedService/
│ │ ├── FrameworkFunctions.json
│ │ ├── Keys.json
│ │ └── SupportDatabase.json
│ ├── pipeline/
│ │ ├── 01-Grandparent.json
│ │ ├── 02-Parent.json
│ │ ├── 03-Child.json
│ │ ├── 04-Infant.json
│ │ ├── Check For Running Pipeline.json
│ │ ├── Email Sender.json
│ │ ├── Intentional Error.json
│ │ ├── Throw Exception.json
│ │ ├── Wait 1.json
│ │ ├── Wait 10.json
│ │ ├── Wait 2.json
│ │ ├── Wait 3.json
│ │ ├── Wait 4.json
│ │ ├── Wait 5.json
│ │ ├── Wait 6.json
│ │ ├── Wait 7.json
│ │ ├── Wait 8.json
│ │ └── Wait 9.json
│ └── trigger/
│ └── FunctionalTestingTrigger.json
├── DeploymentTools/
│ ├── DataFactory/
│ │ ├── DeployProcFwkComponents.ps1
│ │ ├── DeployProcFwkComponents_Old.ps1
│ │ ├── Get Pipelines.ps1
│ │ ├── GlobalVars.ps1
│ │ ├── PopulatePipelinesInDb.ps1
│ │ ├── ProcFwkComponents.json
│ │ └── config-all.csv
│ ├── Deployment.targets
│ └── DeploymentTools.deployproj
├── FactoryTesting/
│ ├── FactoryTesting.csproj
│ ├── Helpers/
│ │ ├── CoverageHelper.cs
│ │ ├── DataFactoryHelper.cs
│ │ ├── DatabaseHelper.cs
│ │ ├── PipelineRunHelper.cs
│ │ └── SettingsHelper.cs
│ ├── Pipelines/
│ │ ├── 01-Grandparent/
│ │ │ ├── Given300WorkerPipelines.cs
│ │ │ ├── Given600WorkerPipelineConcurrentBatches.cs
│ │ │ ├── GivenOneWorkerPipeline.cs
│ │ │ └── GrandparentHelper.cs
│ │ ├── 02-Parent/
│ │ │ ├── Given20ConcurrentBatchesFor1000WorkerPipelines.cs
│ │ │ ├── GivenAlreadyRunning.cs
│ │ │ ├── GivenBatchExecutionsForConcurrentBatches.cs
│ │ │ ├── GivenBatchExecutionsForConcurrentBatchesAlreadyRunning.cs
│ │ │ ├── GivenBatchExecutionsForConcurrentBatchesWithSimpleFailureHandling.cs
│ │ │ ├── GivenBatchExecutionsForConcurrentBatchesWithSimpleFailureHandlingAndRestart.cs
│ │ │ ├── GivenBatchExecutionsForConcurrentBatchesWithSimpleFailureHandlingAndRestartOveride.cs
│ │ │ ├── GivenBatchExecutionsForSingleBatch.cs
│ │ │ ├── GivenCancelledWorkerAndRestart.cs
│ │ │ ├── GivenCancelledWorkerInOneExecutionStage.cs
│ │ │ ├── GivenCancelledWorkerThatBlocks.cs
│ │ │ ├── GivenCancelledWorkerThatDoesntBlock.cs
│ │ │ ├── GivenCleanUpForCancelledWorkerAndRestart.cs
│ │ │ ├── GivenCleanUpForSuccessfulWorkersAndRestart.cs
│ │ │ ├── GivenDependencyChainFailureHandling.cs
│ │ │ ├── GivenDependencyChainFailureHandlingAndRestart.cs
│ │ │ ├── GivenDisabledBatches.cs
│ │ │ ├── GivenDisabledPipelines.cs
│ │ │ ├── GivenDisabledStages.cs
│ │ │ ├── GivenNoErrorsAndSPNStoredInDatabase.cs
│ │ │ ├── GivenNoErrorsAndSPNStoredInKeyVault.cs
│ │ │ ├── GivenNoFailureHandling.cs
│ │ │ ├── GivenNoPipelineParameters.cs
│ │ │ ├── GivenOneExecutionStage.cs
│ │ │ ├── GivenSimpleFailureHandling.cs
│ │ │ ├── GivenSimpleFailureHandlingAndRestart.cs
│ │ │ ├── GivenSimpleFailureHandlingAndRestartOveride.cs
│ │ │ └── ParentHelper.cs
│ │ └── Utilities/
│ │ ├── GivenCheckForRunningPipeline.cs
│ │ ├── GivenEmailSender.cs
│ │ ├── GivenThrowException.cs
│ │ └── UtilitiesHelper.cs
│ ├── dev.runsettings
│ ├── multi.runsettings
│ └── test.runsettings
├── Functions/
│ ├── .vscode/
│ │ ├── extensions.json
│ │ ├── launch.json
│ │ ├── settings.json
│ │ └── tasks.json
│ ├── Functions/
│ │ ├── CancelPipeline.cs
│ │ ├── CheckPipelineStatus.cs
│ │ ├── ExecutePipeline.cs
│ │ ├── GetActivityErrors.cs
│ │ ├── SendEmail.cs
│ │ └── ValidatePipeline.cs
│ ├── Functions.csproj
│ ├── Helpers/
│ │ ├── BodyReader.cs
│ │ ├── InvalidRequestException.cs
│ │ ├── KeyVaultClient.cs
│ │ ├── PipelineRequest.cs
│ │ ├── PipelineRunRequest.cs
│ │ └── SMTPClient.cs
│ ├── Properties/
│ │ └── ServiceDependencies/
│ │ ├── FrameworkSupportFunctions - Zip Deploy/
│ │ │ └── profile.arm.json
│ │ └── FrameworkSupportFunctionsBig - Web Deploy/
│ │ └── profile.arm.json
│ ├── Services/
│ │ ├── AzureDataFactoryService.cs
│ │ ├── AzureSynapseService.cs
│ │ ├── PipelineService.cs
│ │ ├── PipelineServiceType.cs
│ │ └── Returns/
│ │ ├── PipelineDescription.cs
│ │ ├── PipelineErrorDetail.cs
│ │ └── PipelineRunStatus.cs
│ ├── host.json
│ └── template_local.settings.json
├── Images/
│ └── procfwk Designs.vsdx
├── LICENSE
├── MetadataDB/
│ ├── MetadataDB.refactorlog
│ ├── MetadataDB.sqlproj
│ ├── Scripts/
│ │ ├── Alter Database Scale.sql
│ │ ├── Handy Selects.sql
│ │ ├── LogData/
│ │ │ ├── ErrorLogBackup.sql
│ │ │ ├── ErrorLogRestore.sql
│ │ │ ├── ExecutionLogBackup.sql
│ │ │ └── ExecutionLogRestore.sql
│ │ ├── Metadata/
│ │ │ ├── AlertOutcomes.sql
│ │ │ ├── DeleteAll.sql
│ │ │ ├── DropLegacyObjects.sql
│ │ │ ├── DropLegacyTables.sql
│ │ │ ├── Orchestrators.sql
│ │ │ ├── PipelineDependencies.sql
│ │ │ ├── PipelineParams.sql
│ │ │ ├── Pipelines.sql
│ │ │ ├── Properties.sql
│ │ │ ├── RecipientAlertsLink.sql
│ │ │ ├── Recipients.sql
│ │ │ ├── ReplaceDataFactorys.sql
│ │ │ ├── Stages.sql
│ │ │ ├── TransferHelperObjects.sql
│ │ │ └── TransferReportingObjects.sql
│ │ ├── Script.PostDeployment.sql
│ │ ├── Script.PreDeployment.sql
│ │ └── Script.SetLocalAuthenticationDetails.sql
│ ├── Security/
│ │ ├── procfwk.sql
│ │ ├── procfwkHelpers.sql
│ │ ├── procfwkReporting.sql
│ │ ├── procfwkTesting.sql
│ │ └── procfwkuser Role.sql
│ ├── dbo/
│ │ ├── Stored Procedures/
│ │ │ ├── DemoModePrecursor.sql
│ │ │ ├── ExampleCustomExecutionPrecursor.sql
│ │ │ └── FailProcedure.sql
│ │ └── Tables/
│ │ └── ServicePrincipals.sql
│ ├── procfwk/
│ │ ├── Functions/
│ │ │ └── GetPropertyValueInternal.sql
│ │ ├── Stored Procedures/
│ │ │ ├── BatchWrapper.sql
│ │ │ ├── CheckForBlockedPipelines.sql
│ │ │ ├── CheckForEmailAlerts.sql
│ │ │ ├── CheckMetadataIntegrity.sql
│ │ │ ├── CheckPreviousExeuction.sql
│ │ │ ├── CreateNewExecution.sql
│ │ │ ├── ExecutePrecursorProcedure.sql
│ │ │ ├── ExecutionWrapper.sql
│ │ │ ├── GetEmailAlertParts.sql
│ │ │ ├── GetFrameworkOrchestratorDetails.sql
│ │ │ ├── GetPipelineParameters.sql
│ │ │ ├── GetPipelinesInStage.sql
│ │ │ ├── GetPropertyValue.sql
│ │ │ ├── GetStages.sql
│ │ │ ├── GetWorkerAuthDetails.sql
│ │ │ ├── GetWorkerDetailsWrapper.sql
│ │ │ ├── GetWorkerPipelineDetails.sql
│ │ │ ├── ResetExecution.sql
│ │ │ ├── SetErrorLogDetails.sql
│ │ │ ├── SetExecutionBlockDependants.sql
│ │ │ ├── SetLogActivityFailed.sql
│ │ │ ├── SetLogPipelineCancelled.sql
│ │ │ ├── SetLogPipelineChecking.sql
│ │ │ ├── SetLogPipelineFailed.sql
│ │ │ ├── SetLogPipelineLastStatusCheck.sql
│ │ │ ├── SetLogPipelineRunId.sql
│ │ │ ├── SetLogPipelineRunning.sql
│ │ │ ├── SetLogPipelineSuccess.sql
│ │ │ ├── SetLogPipelineUnknown.sql
│ │ │ ├── SetLogPipelineValidating.sql
│ │ │ ├── SetLogStagePreparing.sql
│ │ │ └── UpdateExecutionLog.sql
│ │ ├── Synonyms/
│ │ │ ├── AddPipelineDependant.sql
│ │ │ ├── AddProperty.sql
│ │ │ ├── AddRecipientPipelineAlerts.sql
│ │ │ ├── AddServicePrincipal.sql
│ │ │ ├── AddServicePrincipalUrls.sql
│ │ │ ├── AddServicePrincipalWrapper.sql
│ │ │ ├── AverageStageDuration.sql
│ │ │ ├── CheckForValidURL.sql
│ │ │ ├── CheckStageAndPiplineIntegrity.sql
│ │ │ ├── CompleteExecutionErrorLog.sql
│ │ │ ├── CompleteExecutionLog.sql
│ │ │ ├── CurrentExecutionSummary.sql
│ │ │ ├── DataFactoryDetails.sql
│ │ │ ├── DeleteRecipientAlerts.sql
│ │ │ ├── DeleteServicePrincipal.sql
│ │ │ ├── GetExecutionDetails.sql
│ │ │ ├── LastExecution.sql
│ │ │ ├── LastExecutionSummary.sql
│ │ │ ├── PipelineDependencyChains.sql
│ │ │ ├── PipelineProcesses.sql
│ │ │ ├── ProcessingStageDetails.sql
│ │ │ └── WorkerParallelismOverTime.sql
│ │ ├── Tables/
│ │ │ ├── AlertOutcomes.sql
│ │ │ ├── BatchExecution.sql
│ │ │ ├── BatchStageLink.sql
│ │ │ ├── Batches.sql
│ │ │ ├── CurrentExecution.sql
│ │ │ ├── ErrorLog.sql
│ │ │ ├── ExecutionLog.sql
│ │ │ ├── Orchestrators.sql
│ │ │ ├── PipelineAlertLink.sql
│ │ │ ├── PipelineAuthLink.sql
│ │ │ ├── PipelineDependencies.sql
│ │ │ ├── PipelineParameters.sql
│ │ │ ├── Pipelines.sql
│ │ │ ├── Properties.sql
│ │ │ ├── Recipients.sql
│ │ │ ├── Stages.sql
│ │ │ ├── Subscriptions.sql
│ │ │ └── Tenants.sql
│ │ └── Views/
│ │ ├── CurrentProperties.sql
│ │ ├── DataFactorys.sql
│ │ └── PipelineParameterDataSizes.sql
│ ├── procfwkHelpers/
│ │ ├── Functions/
│ │ │ └── CheckForValidURL.sql
│ │ ├── Stored Procedures/
│ │ │ ├── AddPipelineDependant.sql
│ │ │ ├── AddPipelineViaPowerShell.sql
│ │ │ ├── AddProperty.sql
│ │ │ ├── AddRecipientPipelineAlerts.sql
│ │ │ ├── AddServicePrincipal.sql
│ │ │ ├── AddServicePrincipalUrls.sql
│ │ │ ├── AddServicePrincipalWrapper.sql
│ │ │ ├── CheckStageAndPiplineIntegrity.sql
│ │ │ ├── DeleteMetadataWithIntegrity.sql
│ │ │ ├── DeleteMetadataWithoutIntegrity.sql
│ │ │ ├── DeleteRecipientAlerts.sql
│ │ │ ├── DeleteServicePrincipal.sql
│ │ │ ├── GetExecutionDetails.sql
│ │ │ ├── GetServicePrincipal.sql
│ │ │ ├── SetDefaultAlertOutcomes.sql
│ │ │ ├── SetDefaultBatchStageLink.sql
│ │ │ ├── SetDefaultBatches.sql
│ │ │ ├── SetDefaultOrchestrators.sql
│ │ │ ├── SetDefaultPipelineDependants.sql
│ │ │ ├── SetDefaultPipelineParameters.sql
│ │ │ ├── SetDefaultPipelines.sql
│ │ │ ├── SetDefaultProperties.sql
│ │ │ ├── SetDefaultRecipientPipelineAlerts.sql
│ │ │ ├── SetDefaultRecipients.sql
│ │ │ ├── SetDefaultStages.sql
│ │ │ ├── SetDefaultSubscription.sql
│ │ │ └── SetDefaultTenant.sql
│ │ └── Views/
│ │ └── PipelineDependencyChains.sql
│ ├── procfwkReporting/
│ │ └── Views/
│ │ ├── AverageStageDuration.sql
│ │ ├── CompleteExecutionErrorLog.sql
│ │ ├── CompleteExecutionLog.sql
│ │ ├── CurrentExecutionSummary.sql
│ │ ├── LastExecution.sql
│ │ ├── LastExecutionSummary.sql
│ │ └── WorkerParallelismOverTime.sql
│ └── procfwkTesting/
│ └── Stored Procedures/
│ ├── Add20BatchesFor1000Workers.sql
│ ├── Add300WorkerPipelineBatches.sql
│ ├── Add300WorkerPipelines.sql
│ ├── CleanUpMetadata.sql
│ ├── GetRunIdWhenAvailable.sql
│ └── ResetMetadata.sql
├── MetadataDBTests/
│ ├── MetadataDBTests.sqlproj
│ ├── Scripts/
│ │ ├── Script.PostDeployment.sql
│ │ └── Script.PreDeployment.sql
│ ├── _TestClasses/
│ │ ├── procfwk_GetPropertyValue.sql
│ │ └── procfwk_GetPropertyValueInternal.sql
│ ├── procfwk_GetPropertyValue/
│ │ ├── setup.sql
│ │ ├── test WHEN property does not exist THEN error raised.sql
│ │ ├── test WHEN property exists THEN property value returned.sql
│ │ ├── test WHEN property invalidated THEN error raised.sql
│ │ └── test WHEN property name is null THEN error raised.sql
│ ├── procfwk_GetPropertyValueInternal/
│ │ ├── test WHEN property does not exist THEN empty string returned.sql
│ │ ├── test WHEN property exists THEN property value returned.sql
│ │ └── test WHEN property name is null THEN empty string returned.sql
│ └── tSQLt.dacpac
├── Notebooks/
│ ├── Databricks - Throw Exception.scala
│ └── Metadata Guide and Handy Code Snippets.ipynb
├── ProcessingFramework.sln
├── README.md
├── Reporting/
│ ├── PowerBI/
│ │ ├── Executions Overview.pbit
│ │ └── Executions Overview.pbix
│ └── Scoping-Template-LDI-Blank.docx
└── Synapse/
├── dataflow/
│ └── MDF _Order_Count.json
├── dataset/
│ └── GetSetMetadata.json
├── factory/
│ └── FrameworkFactory.json
├── integrationRuntime/
│ └── AutoResolveIntegrationRuntime.json
├── linkedService/
│ ├── FrameworkFunctions.json
│ ├── Keys.json
│ ├── SupportDatabase.json
│ ├── covid-tracking.json
│ ├── procfwkforsynapse-WorkspaceDefaultSqlServer.json
│ └── procfwkforsynapse-WorkspaceDefaultStorage.json
├── notebook/
│ └── Getting Started with Delta Lake.json
├── pipeline/
│ ├── 01-Grandparent.json
│ ├── 02-Parent.json
│ ├── 03-Child.json
│ ├── 04-Infant.json
│ ├── Check For Running Pipeline.json
│ ├── Email Sender.json
│ ├── Intentional Error.json
│ ├── Throw Exception.json
│ ├── Wait 1.json
│ ├── Wait 10.json
│ ├── Wait 2.json
│ ├── Wait 3.json
│ ├── Wait 4.json
│ ├── Wait 5.json
│ ├── Wait 6.json
│ ├── Wait 7.json
│ ├── Wait 8.json
│ └── Wait 9.json
└── trigger/
└── FunctionalTestingTrigger.json
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitattributes
================================================
# Auto detect text files and perform LF normalization
* text=auto
================================================
FILE: .github/FUNDING.yml
================================================
# These are supported funding model platforms
github: [mrpaulandrew]
================================================
FILE: .github/ISSUE_TEMPLATE/bug-found.md
================================================
---
name: Bug Found
about: Tell me about the bug you've found in the procfwk please
title: ''
labels: bug
assignees: mrpaulandrew
---
**Describe the bug**
A clear and concise description of what the bug is.
**Affected services**
Which resource within the processing framework does this affect?
* Data Factory/Synapse
* SQL Database
* Functions
* All of them
* Other
**To Reproduce**
Steps to reproduce the behavior:
1. Go to '...'
2. Click on '....'
3. Scroll down to '....'
4. See error
**Expected behaviour**
A clear and concise description of what you expected to happen.
**Screenshots**
If applicable, add screenshots to help explain your problem.
**Additional context**
Add any other context about the problem here.
================================================
FILE: .github/ISSUE_TEMPLATE/feature-request.md
================================================
---
name: Feature Request
about: Tell me about your idea to enhance the procfwk please
title: ''
labels: enhancement
assignees: mrpaulandrew
---
**Share Your Idea**
All features considered.
================================================
FILE: .github/ISSUE_TEMPLATE/help---support-request.md
================================================
---
name: Help & Support Request
about: Tell me about the problem or error you are facing when using the procfwk
title: ''
labels: help wanted
assignees: mrpaulandrew
---
**Describe the error**
A clear and concise description of what the bug is.
**Error message**
An output of the error message presented.
**Affected services**
Which resource within the processing framework does this affect?
* Data Factory/Synapse
* SQL Database
* Functions
* All of them
* Other
**To Reproduce**
Steps to reproduce the behavior:
1. Go to '...'
2. Click on '....'
3. Scroll down to '....'
4. See error
**Screenshots**
If applicable, add screenshots to help explain your problem.
================================================
FILE: .gitignore
================================================
################################################################################
# This .gitignore file was automatically created by Microsoft(R) Visual Studio.
################################################################################
## Ignore Visual Studio temporary files, build results, and
## files generated by popular Visual Studio add-ons.
## https://stackoverflow.com/questions/30868544/gitignore-wont-ignore-vs-folder-for-visual-studio-2015-rc-on-windows7-8/39520183
# User-specific files
*.suo
*.user
*.userosscache
*.sln.docstates
# User-specific files (MonoDevelop/Xamarin Studio)
*.userprefs
# Build results
[Dd]ebug/
[Dd]ebugPublic/
[Rr]elease/
[Rr]eleases/
x64/
x86/
bld/
[Bb]in/
[Oo]bj/
[Ll]og/
# Visual Studio 2015 cache/options directory
.vs/
# Uncomment if you have tasks that create the project's static files in wwwroot
#wwwroot/
# MSTest test Results
[Tt]est[Rr]esult*/
[Bb]uild[Ll]og.*
# NUNIT
*.VisualState.xml
TestResult.xml
# Build Results of an ATL Project
[Dd]ebugPS/
[Rr]eleasePS/
dlldata.c
# DNX
project.lock.json
artifacts/
*_i.c
*_p.c
*_i.h
*.ilk
*.meta
*.obj
*.pch
*.pdb
*.pgc
*.pgd
*.rsp
*.sbr
*.tlb
*.tli
*.tlh
*.tmp
*.tmp_proj
*.log
*.vspscc
*.vssscc
.builds
*.pidb
*.svclog
*.scc
# Chutzpah Test files
_Chutzpah*
# Visual C++ cache files
ipch/
*.aps
*.ncb
*.opendb
*.opensdf
*.sdf
*.cachefile
# Visual Studio profiler
*.psess
*.vsp
*.vspx
*.sap
# TFS 2012 Local Workspace
$tf/
# Guidance Automation Toolkit
*.gpState
# ReSharper is a .NET coding add-in
_ReSharper*/
*.[Rr]e[Ss]harper
*.DotSettings.user
# JustCode is a .NET coding add-in
.JustCode
# TeamCity is a build add-in
_TeamCity*
# DotCover is a Code Coverage Tool
*.dotCover
# NCrunch
_NCrunch_*
.*crunch*.local.xml
nCrunchTemp_*
# MightyMoose
*.mm.*
AutoTest.Net/
# Web workbench (sass)
.sass-cache/
# Installshield output folder
[Ee]xpress/
# DocProject is a documentation generator add-in
DocProject/buildhelp/
DocProject/Help/*.HxT
DocProject/Help/*.HxC
DocProject/Help/*.hhc
DocProject/Help/*.hhk
DocProject/Help/*.hhp
DocProject/Help/Html2
DocProject/Help/html
# Click-Once directory
publish/
# Publish Web Output
*.[Pp]ublish.xml
*.azurePubxml
# TODO: Comment the next line if you want to checkin your web deploy settings
# but database connection strings (with potential passwords) will be unencrypted
*.pubxml
*.publishproj
# NuGet Packages
*.nupkg
# The packages folder can be ignored because of Package Restore
**/packages/*
# except build/, which is used as an MSBuild target.
!**/packages/build/
# Uncomment if necessary however generally it will be regenerated when needed
#!**/packages/repositories.config
# NuGet v3's project.json files produces more ignoreable files
*.nuget.props
*.nuget.targets
# Microsoft Azure Build Output
csx/
*.build.csdef
# Microsoft Azure Emulator
ecf/
rcf/
# Windows Store app package directories and files
AppPackages/
BundleArtifacts/
Package.StoreAssociation.xml
_pkginfo.txt
# Visual Studio cache files
# files ending in .cache can be ignored
*.[Cc]ache
# but keep track of directories ending in .cache
!*.[Cc]ache/
# Others
ClientBin/
~$*
*~
*.dbmdl
*.dbproj.schemaview
*.pfx
*.publishsettings
node_modules/
orleans.codegen.cs
# Since there are multiple workflows, uncomment next line to ignore bower_components
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
#bower_components/
# RIA/Silverlight projects
Generated_Code/
# Backup & report files from converting an old project file
# to a newer Visual Studio version. Backup files are not needed,
# because we have git ;-)
_UpgradeReport_Files/
Backup*/
UpgradeLog*.XML
UpgradeLog*.htm
# SQL Server files
*.mdf
*.ldf
*.jfm
# Business Intelligence projects
*.rdl.data
*.bim.layout
*.bim_*.settings
# Microsoft Fakes
FakesAssemblies/
# GhostDoc plugin setting file
*.GhostDoc.xml
# Node.js Tools for Visual Studio
.ntvs_analysis.dat
# Visual Studio 6 build log
*.plg
# Visual Studio 6 workspace options file
*.opt
# Visual Studio LightSwitch build output
**/*.HTMLClient/GeneratedArtifacts
**/*.DesktopClient/GeneratedArtifacts
**/*.DesktopClient/ModelManifest.xml
**/*.Server/GeneratedArtifacts
**/*.Server/ModelManifest.xml
_Pvt_Extensions
# Paket dependency manager
.paket/paket.exe
paket-files/
# FAKE - F# Make
.fake/
# JetBrains Rider
.idea/
*.sln.iml
/PipelineExecutor/local.settings.json
/Functions/local.settings.json
/docs
================================================
FILE: .vscode/extensions.json
================================================
{
"recommendations": [
"ms-azuretools.vscode-azurefunctions",
"ms-vscode.csharp"
]
}
================================================
FILE: .vscode/launch.json
================================================
{
"version": "0.2.0",
"configurations": [
{
"name": "Attach to .NET Functions",
"type": "coreclr",
"request": "attach",
"processId": "${command:azureFunctions.pickProcess}"
}
]
}
================================================
FILE: .vscode/settings.json
================================================
{
"azureFunctions.deploySubpath": "PipelineExecutor/bin/Release/netcoreapp3.1/publish",
"azureFunctions.projectLanguage": "C#",
"azureFunctions.projectRuntime": "~3",
"debug.internalConsoleOptions": "neverOpen",
"azureFunctions.preDeployTask": "publish"
}
================================================
FILE: .vscode/tasks.json
================================================
{
"version": "2.0.0",
"tasks": [
{
"label": "clean",
"command": "dotnet",
"args": [
"clean",
"/property:GenerateFullPaths=true",
"/consoleloggerparameters:NoSummary"
],
"type": "process",
"problemMatcher": "$msCompile",
"options": {
"cwd": "${workspaceFolder}/PipelineExecutor"
}
},
{
"label": "build",
"command": "dotnet",
"args": [
"build",
"/property:GenerateFullPaths=true",
"/consoleloggerparameters:NoSummary"
],
"type": "process",
"dependsOn": "clean",
"group": {
"kind": "build",
"isDefault": true
},
"problemMatcher": "$msCompile",
"options": {
"cwd": "${workspaceFolder}/PipelineExecutor"
}
},
{
"label": "clean release",
"command": "dotnet",
"args": [
"clean",
"--configuration",
"Release",
"/property:GenerateFullPaths=true",
"/consoleloggerparameters:NoSummary"
],
"type": "process",
"problemMatcher": "$msCompile",
"options": {
"cwd": "${workspaceFolder}/PipelineExecutor"
}
},
{
"label": "publish",
"command": "dotnet",
"args": [
"publish",
"--configuration",
"Release",
"/property:GenerateFullPaths=true",
"/consoleloggerparameters:NoSummary"
],
"type": "process",
"dependsOn": "clean release",
"problemMatcher": "$msCompile",
"options": {
"cwd": "${workspaceFolder}/PipelineExecutor"
}
},
{
"type": "func",
"dependsOn": "build",
"options": {
"cwd": "${workspaceFolder}/PipelineExecutor/bin/Debug/netcoreapp3.1"
},
"command": "host start",
"isBackground": true,
"problemMatcher": "$func-watch"
}
]
}
================================================
FILE: ARM Templates/Data Factory/v1.0 Export.json
================================================
{
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"factoryName": {
"type": "string",
"metadata": "Data Factory name"
},
"Keys_properties_typeProperties_baseUrl": {
"type": "string"
},
"SupportDatabase_properties_typeProperties_connectionString_secretName": {
"type": "string"
}
},
"variables": {
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
},
"resources": [
{
"name": "[concat(parameters('factoryName'), '/BootStrap')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Get Stages",
"type": "Lookup",
"dependsOn": [
{
"activity": "Create New Execution",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[[procfwk].[GetProcessStages]",
"storedProcedureParameters": {
"ExecutionId": {
"type": "Guid",
"value": {
"value": "@activity('Create New Execution').output.firstRow.ExecutionId",
"type": "Expression"
}
}
},
"queryTimeout": "02:00:00"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
},
"firstRowOnly": false
}
},
{
"name": "Execute Stages",
"type": "ForEach",
"dependsOn": [
{
"activity": "Get Stages",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@activity('Get Stages').output.value",
"type": "Expression"
},
"isSequential": true,
"activities": [
{
"name": "Stage Executor",
"type": "ExecutePipeline",
"dependsOn": [
{
"activity": "Log Stage Start",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"pipeline": {
"referenceName": "Executor",
"type": "PipelineReference"
},
"waitOnCompletion": true,
"parameters": {
"StageId": {
"value": "@item().StageId",
"type": "Expression"
},
"ExecutionId": {
"value": "@activity('Create New Execution').output.firstRow.ExecutionId",
"type": "Expression"
}
}
}
},
{
"name": "Log Stage Start",
"type": "SqlServerStoredProcedure",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[[procfwk].[SetLogStageStart]",
"storedProcedureParameters": {
"ExecutionId": {
"value": {
"value": "@activity('Create New Execution').output.firstRow.ExecutionId",
"type": "Expression"
},
"type": "Guid"
},
"StageId": {
"value": {
"value": "@item().StageId",
"type": "Expression"
},
"type": "Int32"
}
}
},
"linkedServiceName": {
"referenceName": "SupportDatabase",
"type": "LinkedServiceReference"
}
}
]
}
},
{
"name": "Create New Execution",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[[procfwk].[CreateNewExecution]",
"queryTimeout": "02:00:00"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
}
}
},
{
"name": "Update Execution Log",
"type": "SqlServerStoredProcedure",
"dependsOn": [
{
"activity": "Execute Stages",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[[procfwk].[UpdateExecutionLog]"
},
"linkedServiceName": {
"referenceName": "SupportDatabase",
"type": "LinkedServiceReference"
}
}
],
"folder": {
"name": "Framework"
},
"annotations": []
},
"dependsOn": [
"[concat(variables('factoryId'), '/datasets/GetSetMetadata')]",
"[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]",
"[concat(variables('factoryId'), '/pipelines/Executor')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/Child 1')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Wait1",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 1
}
}
],
"folder": {
"name": "Child Pipelines"
},
"annotations": []
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/Child 2')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Wait1",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 1
}
}
],
"folder": {
"name": "Child Pipelines"
},
"annotations": []
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/Child 3')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Wait1",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 1
}
}
],
"folder": {
"name": "Child Pipelines"
},
"annotations": []
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/Child 4')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Wait1",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 1
}
}
],
"folder": {
"name": "Child Pipelines"
},
"annotations": []
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/Child 5')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Wait1",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 1
}
}
],
"folder": {
"name": "Child Pipelines"
},
"annotations": []
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/Child 6')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Wait1",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 1
}
}
],
"folder": {
"name": "Child Pipelines"
},
"annotations": []
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/Executor')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Get Pipelines",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[[procfwk].[GetPipelinesInStage]",
"storedProcedureParameters": {
"StageId": {
"type": "Int32",
"value": {
"value": "@pipeline().parameters.StageId",
"type": "Expression"
}
}
},
"queryTimeout": "02:00:00"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
},
"firstRowOnly": false
}
},
{
"name": "Execute Pipelines",
"type": "ForEach",
"dependsOn": [
{
"activity": "Get Pipelines",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@activity('Get Pipelines').output.value",
"type": "Expression"
},
"activities": [
{
"name": "Execute Pipeline",
"type": "AzureFunctionActivity",
"dependsOn": [
{
"activity": "Set Body",
"dependencyConditions": [
"Succeeded"
]
},
{
"activity": "Log Pipeline Running",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"functionName": "ExecutePipeline",
"method": "POST",
"headers": {},
"body": {
"value": "@variables('FunctionBody')",
"type": "Expression"
}
},
"linkedServiceName": {
"referenceName": "PipelineExecutor",
"type": "LinkedServiceReference"
}
},
{
"name": "Set Body",
"type": "SetVariable",
"dependsOn": [
{
"activity": "Get Pipeline Params",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"variableName": "FunctionBody",
"value": {
"value": "@concat('\n{\n\t\"tenantId\": \"1234-1234-1234-1234-1234\",\n\t\"applicationId\": \"1234-1234-1234-1234-1234\",\n\t\"authenticationKey\": \"Passw0rd123!\",\n\t\"subscriptionId\": \"1234-1234-1234-1234-1234\",\n\t\"resourceGroup\": \"Demos\",\n\t\"factoryName\": \"FunFactory\",\n\t\"pipelineName\": \"',item().PipelineName,'\"',activity('Get Pipeline Params').output.firstRow.Params,'\n}')",
"type": "Expression"
}
}
},
{
"name": "Get Pipeline Params",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[[procfwk].[GetPipelineParameters]",
"storedProcedureParameters": {
"PipelineId": {
"type": "Int32",
"value": {
"value": "@item().PipelineId",
"type": "Expression"
}
}
},
"queryTimeout": "02:00:00"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
}
}
},
{
"name": "Log Pipeline Running",
"type": "SqlServerStoredProcedure",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[[procfwk].[SetLogPipelineRunning]",
"storedProcedureParameters": {
"ExecutionId": {
"value": {
"value": "@pipeline().parameters.ExecutionId",
"type": "Expression"
},
"type": "Guid"
},
"PipelineId": {
"value": {
"value": "@item().PipelineId",
"type": "Expression"
},
"type": "Int32"
},
"StageId": {
"value": {
"value": "@pipeline().parameters.StageId",
"type": "Expression"
},
"type": "Int32"
}
}
},
"linkedServiceName": {
"referenceName": "SupportDatabase",
"type": "LinkedServiceReference"
}
},
{
"name": "Log Pipeline Success",
"type": "SqlServerStoredProcedure",
"dependsOn": [
{
"activity": "Execute Pipeline",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]",
"storedProcedureParameters": {
"ExecutionId": {
"value": {
"value": "@pipeline().parameters.ExecutionId",
"type": "Expression"
},
"type": "Guid"
},
"PipelineId": {
"value": {
"value": "@item().PipelineId",
"type": "Expression"
},
"type": "Int32"
},
"StageId": {
"value": {
"value": "@pipeline().parameters.StageId",
"type": "Expression"
},
"type": "Int32"
}
}
},
"linkedServiceName": {
"referenceName": "SupportDatabase",
"type": "LinkedServiceReference"
}
},
{
"name": "Log Pipeline Failure",
"type": "SqlServerStoredProcedure",
"dependsOn": [
{
"activity": "Execute Pipeline",
"dependencyConditions": [
"Failed"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[[procfwk].[SetLogPipelineFailed]",
"storedProcedureParameters": {
"ExecutionId": {
"value": {
"value": "@pipeline().parameters.ExecutionId",
"type": "Expression"
},
"type": "Guid"
},
"PipelineId": {
"value": {
"value": "@item().PipelineId",
"type": "Expression"
},
"type": "Int32"
},
"StageId": {
"value": {
"value": "@pipeline().parameters.StageId",
"type": "Expression"
},
"type": "Int32"
}
}
},
"linkedServiceName": {
"referenceName": "SupportDatabase",
"type": "LinkedServiceReference"
}
}
]
}
}
],
"parameters": {
"StageId": {
"type": "int"
},
"ExecutionId": {
"type": "string"
}
},
"variables": {
"FunctionBody": {
"type": "String"
}
},
"folder": {
"name": "Framework"
},
"annotations": []
},
"dependsOn": [
"[concat(variables('factoryId'), '/datasets/GetSetMetadata')]",
"[concat(variables('factoryId'), '/linkedServices/PipelineExecutor')]",
"[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/Grandparent')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Framework Processing",
"type": "ExecutePipeline",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"pipeline": {
"referenceName": "BootStrap",
"type": "PipelineReference"
},
"waitOnCompletion": true,
"parameters": {}
}
}
],
"annotations": []
},
"dependsOn": [
"[concat(variables('factoryId'), '/pipelines/BootStrap')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/GetSetMetadata')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "SupportDatabase",
"type": "LinkedServiceReference"
},
"annotations": [],
"type": "AzureSqlTable",
"schema": [],
"typeProperties": {}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/Keys')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"annotations": [],
"type": "AzureKeyVault",
"typeProperties": {
"baseUrl": "[parameters('Keys_properties_typeProperties_baseUrl')]"
}
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/PipelineExecutor')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"annotations": [],
"type": "AzureFunction",
"typeProperties": {
"functionAppUrl": "https://pipelineexecutor.azurewebsites.net",
"functionKey": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "Keys",
"type": "LinkedServiceReference"
},
"secretName": "ExecutorFunctionAppKey"
}
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/Keys')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/SupportDatabase')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"annotations": [],
"type": "AzureSqlDatabase",
"typeProperties": {
"connectionString": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "Keys",
"type": "LinkedServiceReference"
},
"secretName": "[parameters('SupportDatabase_properties_typeProperties_connectionString_secretName')]"
}
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/Keys')]"
]
}
]
}
================================================
FILE: ARM Templates/Data Factory/v1.1 Export.json
================================================
{
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"factoryName": {
"type": "string",
"metadata": "Data Factory name",
"defaultValue": ""
},
"Keys_properties_typeProperties_baseUrl": {
"type": "string",
"defaultValue": ""
},
"SupportDatabase_properties_typeProperties_connectionString_secretName": {
"type": "string",
"defaultValue": ""
}
},
"variables": {
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
},
"resources": [
{
"name": "[concat(parameters('factoryName'), '/BootStrap')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Get Stages",
"type": "Lookup",
"dependsOn": [
{
"activity": "Create New Execution",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[[procfwk].[GetProcessStages]",
"storedProcedureParameters": {
"ExecutionId": {
"type": "Guid",
"value": {
"value": "@activity('Create New Execution').output.firstRow.ExecutionId",
"type": "Expression"
}
}
},
"queryTimeout": "02:00:00"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
},
"firstRowOnly": false
}
},
{
"name": "Execute Stages",
"type": "ForEach",
"dependsOn": [
{
"activity": "Get Stages",
"dependencyConditions": [
"Succeeded"
]
},
{
"activity": "Get Tenant Id",
"dependencyConditions": [
"Succeeded"
]
},
{
"activity": "Get Subscription Id",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@activity('Get Stages').output.value",
"type": "Expression"
},
"isSequential": true,
"activities": [
{
"name": "Stage Executor",
"type": "ExecutePipeline",
"dependsOn": [
{
"activity": "Log Stage Start",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"pipeline": {
"referenceName": "Executor",
"type": "PipelineReference"
},
"waitOnCompletion": true,
"parameters": {
"StageId": {
"value": "@item().StageId",
"type": "Expression"
},
"ExecutionId": {
"value": "@activity('Create New Execution').output.firstRow.ExecutionId",
"type": "Expression"
},
"TenantId": {
"value": "@activity('Get Tenant Id').output.firstRow.PropertyValue",
"type": "Expression"
},
"SubscriptionId": {
"value": "@activity('Get Subscription Id').output.firstRow.PropertyValue",
"type": "Expression"
}
}
}
},
{
"name": "Log Stage Start",
"type": "SqlServerStoredProcedure",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[[procfwk].[SetLogStageStart]",
"storedProcedureParameters": {
"ExecutionId": {
"value": {
"value": "@activity('Create New Execution').output.firstRow.ExecutionId",
"type": "Expression"
},
"type": "Guid"
},
"StageId": {
"value": {
"value": "@item().StageId",
"type": "Expression"
},
"type": "Int32"
}
}
},
"linkedServiceName": {
"referenceName": "SupportDatabase",
"type": "LinkedServiceReference"
}
}
]
}
},
{
"name": "Create New Execution",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[[procfwk].[CreateNewExecution]",
"queryTimeout": "02:00:00"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
}
}
},
{
"name": "Update Execution Log",
"type": "SqlServerStoredProcedure",
"dependsOn": [
{
"activity": "Execute Stages",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[[procfwk].[UpdateExecutionLog]"
},
"linkedServiceName": {
"referenceName": "SupportDatabase",
"type": "LinkedServiceReference"
}
},
{
"name": "Get Tenant Id",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]",
"storedProcedureParameters": {
"PropertyName": {
"type": "String",
"value": "TenantId"
}
},
"queryTimeout": "02:00:00"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
},
"firstRowOnly": true
}
},
{
"name": "Get Subscription Id",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": true
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]",
"storedProcedureParameters": {
"PropertyName": {
"type": "String",
"value": "SubscriptionId"
}
},
"queryTimeout": "02:00:00"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
}
}
}
],
"folder": {
"name": "Framework"
},
"annotations": []
},
"dependsOn": [
"[concat(variables('factoryId'), '/datasets/GetSetMetadata')]",
"[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]",
"[concat(variables('factoryId'), '/pipelines/Executor')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/Child 1')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Wait1",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 1
}
}
],
"folder": {
"name": "Child Pipelines"
},
"annotations": []
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/Child 2')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Wait1",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 1
}
}
],
"folder": {
"name": "Child Pipelines"
},
"annotations": []
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/Child 3')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Wait1",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 1
}
}
],
"folder": {
"name": "Child Pipelines"
},
"annotations": []
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/Child 4')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Wait1",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 1
}
}
],
"folder": {
"name": "Child Pipelines"
},
"annotations": []
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/Child 5')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Wait1",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 1
}
}
],
"folder": {
"name": "Child Pipelines"
},
"annotations": []
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/Child 6')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Wait1",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 1
}
}
],
"folder": {
"name": "Child Pipelines"
},
"annotations": []
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/Executor')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Get Pipelines",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[[procfwk].[GetPipelinesInStage]",
"storedProcedureParameters": {
"StageId": {
"type": "Int32",
"value": {
"value": "@pipeline().parameters.StageId",
"type": "Expression"
}
}
},
"queryTimeout": "02:00:00"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
},
"firstRowOnly": false
}
},
{
"name": "Execute Pipelines",
"type": "ForEach",
"dependsOn": [
{
"activity": "Get Pipelines",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@activity('Get Pipelines').output.value",
"type": "Expression"
},
"activities": [
{
"name": "Execute Pipeline",
"type": "AzureFunctionActivity",
"dependsOn": [
{
"activity": "Log Pipeline Running",
"dependencyConditions": [
"Succeeded"
]
},
{
"activity": "Get Pipeline Params",
"dependencyConditions": [
"Succeeded"
]
},
{
"activity": "Get SPN Details",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"functionName": "ExecutePipeline",
"method": "POST",
"headers": {},
"body": {
"value": "@concat('\n{\n\t\"tenantId\": \"',pipeline().parameters.TenantId,'\",\n\t\"applicationId\": \"',activity('Get SPN Details').output.firstRow.Id,'\",\n\t\"authenticationKey\": \"',activity('Get SPN Details').output.firstRow.Secret,'\",\n\t\"subscriptionId\": \"',pipeline().parameters.SubscriptionId,'\",\n\t\"resourceGroup\": \"',item().ResourceGroupName,'\",\n\t\"factoryName\": \"',item().DataFactoryName,'\",\n\t\"pipelineName\": \"',item().PipelineName,'\"',activity('Get Pipeline Params').output.firstRow.Params,'\n}')",
"type": "Expression"
}
},
"linkedServiceName": {
"referenceName": "PipelineExecutor",
"type": "LinkedServiceReference"
}
},
{
"name": "Get Pipeline Params",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[[procfwk].[GetPipelineParameters]",
"storedProcedureParameters": {
"PipelineId": {
"type": "Int32",
"value": {
"value": "@item().PipelineId",
"type": "Expression"
}
}
},
"queryTimeout": "02:00:00"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
}
}
},
{
"name": "Log Pipeline Running",
"type": "SqlServerStoredProcedure",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[[procfwk].[SetLogPipelineRunning]",
"storedProcedureParameters": {
"ExecutionId": {
"value": {
"value": "@pipeline().parameters.ExecutionId",
"type": "Expression"
},
"type": "Guid"
},
"PipelineId": {
"value": {
"value": "@item().PipelineId",
"type": "Expression"
},
"type": "Int32"
},
"StageId": {
"value": {
"value": "@pipeline().parameters.StageId",
"type": "Expression"
},
"type": "Int32"
}
}
},
"linkedServiceName": {
"referenceName": "SupportDatabase",
"type": "LinkedServiceReference"
}
},
{
"name": "Log Pipeline Success",
"type": "SqlServerStoredProcedure",
"dependsOn": [
{
"activity": "Execute Pipeline",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]",
"storedProcedureParameters": {
"ExecutionId": {
"value": {
"value": "@pipeline().parameters.ExecutionId",
"type": "Expression"
},
"type": "Guid"
},
"PipelineId": {
"value": {
"value": "@item().PipelineId",
"type": "Expression"
},
"type": "Int32"
},
"StageId": {
"value": {
"value": "@pipeline().parameters.StageId",
"type": "Expression"
},
"type": "Int32"
}
}
},
"linkedServiceName": {
"referenceName": "SupportDatabase",
"type": "LinkedServiceReference"
}
},
{
"name": "Log Pipeline Failure",
"type": "SqlServerStoredProcedure",
"dependsOn": [
{
"activity": "Execute Pipeline",
"dependencyConditions": [
"Failed"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[[procfwk].[SetLogPipelineFailed]",
"storedProcedureParameters": {
"ExecutionId": {
"value": {
"value": "@pipeline().parameters.ExecutionId",
"type": "Expression"
},
"type": "Guid"
},
"PipelineId": {
"value": {
"value": "@item().PipelineId",
"type": "Expression"
},
"type": "Int32"
},
"StageId": {
"value": {
"value": "@pipeline().parameters.StageId",
"type": "Expression"
},
"type": "Int32"
}
}
},
"linkedServiceName": {
"referenceName": "SupportDatabase",
"type": "LinkedServiceReference"
}
},
{
"name": "Get SPN Details",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[[procfwk].[GetServicePrincipal]",
"storedProcedureParameters": {
"DataFactory": {
"type": "String",
"value": {
"value": "@item().DataFactoryName",
"type": "Expression"
}
},
"PipelineName": {
"type": "String",
"value": {
"value": "@item().PipelineName",
"type": "Expression"
}
}
},
"queryTimeout": "02:00:00"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
}
}
}
]
}
}
],
"parameters": {
"StageId": {
"type": "int"
},
"ExecutionId": {
"type": "string"
},
"TenantId": {
"type": "string"
},
"SubscriptionId": {
"type": "string"
}
},
"variables": {
"FunctionBody": {
"type": "String"
}
},
"folder": {
"name": "Framework"
},
"annotations": []
},
"dependsOn": [
"[concat(variables('factoryId'), '/datasets/GetSetMetadata')]",
"[concat(variables('factoryId'), '/linkedServices/PipelineExecutor')]",
"[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/Grandparent')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Framework Processing",
"type": "ExecutePipeline",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"pipeline": {
"referenceName": "BootStrap",
"type": "PipelineReference"
},
"waitOnCompletion": true,
"parameters": {}
}
}
],
"annotations": []
},
"dependsOn": [
"[concat(variables('factoryId'), '/pipelines/BootStrap')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/GetSetMetadata')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "SupportDatabase",
"type": "LinkedServiceReference"
},
"annotations": [],
"type": "AzureSqlTable",
"schema": [],
"typeProperties": {}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/Keys')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"annotations": [],
"type": "AzureKeyVault",
"typeProperties": {
"baseUrl": "[parameters('Keys_properties_typeProperties_baseUrl')]"
}
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/PipelineExecutor')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"annotations": [],
"type": "AzureFunction",
"typeProperties": {
"functionAppUrl": "https://pipelineexecutor.azurewebsites.net",
"functionKey": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "Keys",
"type": "LinkedServiceReference"
},
"secretName": "ExecutorFunctionAppKey"
}
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/Keys')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/SupportDatabase')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"annotations": [],
"type": "AzureSqlDatabase",
"typeProperties": {
"connectionString": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "Keys",
"type": "LinkedServiceReference"
},
"secretName": "[parameters('SupportDatabase_properties_typeProperties_connectionString_secretName')]"
}
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/Keys')]"
]
}
]
}
================================================
FILE: ARM Templates/Data Factory/v1.2 Export.json
================================================
{
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"factoryName": {
"type": "string",
"metadata": "Data Factory name",
"defaultValue": ""
},
"Keys_properties_typeProperties_baseUrl": {
"type": "string",
"defaultValue": ""
},
"SupportDatabase_properties_typeProperties_connectionString_secretName": {
"type": "string",
"defaultValue": ""
}
},
"variables": {
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
},
"resources": [
{
"name": "[concat(parameters('factoryName'), '/01 Grandparent')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Framework Processing",
"type": "ExecutePipeline",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"pipeline": {
"referenceName": "02 Parent",
"type": "PipelineReference"
},
"waitOnCompletion": true,
"parameters": {}
}
}
],
"folder": {
"name": "_ProcFwk"
},
"annotations": []
},
"dependsOn": [
"[concat(variables('factoryId'), '/pipelines/02 Parent')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/02 Parent')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"description": "ADF.procfwk parent pipeline used to bootstrap the orchestration framework in perform the first level ForEach calls in sequence for the metadata stages.",
"activities": [
{
"name": "Get Stages",
"description": "Returns a distinct list of execution stages within the framework metadata.",
"type": "Lookup",
"dependsOn": [
{
"activity": "Execution Wrapper",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[[procfwk].[GetStages]",
"storedProcedureParameters": {
"ExecutionId": {
"type": "Guid",
"value": {
"value": "@activity('Execution Wrapper').output.firstRow.ExecutionId",
"type": "Expression"
}
}
},
"queryTimeout": "02:00:00"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
},
"firstRowOnly": false
}
},
{
"name": "Execute Stages",
"description": "Top level ForEach to sequentially call all processing stages within the framework metadata. Items for iteration passed from the Get Stages lookup activity.",
"type": "ForEach",
"dependsOn": [
{
"activity": "Get Stages",
"dependencyConditions": [
"Succeeded"
]
},
{
"activity": "Get Tenant Id",
"dependencyConditions": [
"Succeeded"
]
},
{
"activity": "Get Subscription Id",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@activity('Get Stages').output.value",
"type": "Expression"
},
"isSequential": true,
"activities": [
{
"name": "Stage Executor",
"description": "Call to the framework generic child pipeline for a given execution stage.",
"type": "ExecutePipeline",
"dependsOn": [
{
"activity": "Log Stage Preparing",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"pipeline": {
"referenceName": "03 Child",
"type": "PipelineReference"
},
"waitOnCompletion": true,
"parameters": {
"StageId": {
"value": "@item().StageId",
"type": "Expression"
},
"ExecutionId": {
"value": "@activity('Execution Wrapper').output.firstRow.ExecutionId",
"type": "Expression"
},
"TenantId": {
"value": "@activity('Get Tenant Id').output.firstRow.PropertyValue",
"type": "Expression"
},
"SubscriptionId": {
"value": "@activity('Get Subscription Id').output.firstRow.PropertyValue",
"type": "Expression"
}
}
}
},
{
"name": "Log Stage Preparing",
"description": "Update the current execution table flagging all pipelines within the stage as preparing.",
"type": "SqlServerStoredProcedure",
"dependsOn": [
{
"activity": "Check for Blockers",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[[procfwk].[SetLogStagePreparing]",
"storedProcedureParameters": {
"ExecutionId": {
"value": {
"value": "@activity('Execution Wrapper').output.firstRow.ExecutionId",
"type": "Expression"
},
"type": "Guid"
},
"StageId": {
"value": {
"value": "@item().StageId",
"type": "Expression"
},
"type": "Int32"
}
}
},
"linkedServiceName": {
"referenceName": "SupportDatabase",
"type": "LinkedServiceReference"
}
},
{
"name": "Check for Blockers",
"description": "Used to double check and stop the next execution stage if failures and blockers have be incurred. Without this step processing would continue regardless of upstream failures.",
"type": "SqlServerStoredProcedure",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[[procfwk].[CheckForBlockedPipelines]",
"storedProcedureParameters": {
"StageId": {
"value": {
"value": "@item().StageId",
"type": "Expression"
},
"type": "Int32"
}
}
},
"linkedServiceName": {
"referenceName": "SupportDatabase",
"type": "LinkedServiceReference"
}
}
]
}
},
{
"name": "Execution Wrapper",
"description": "Wrapper to reset and restart processing or create a completely new execution instance of the framework metadata.",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[[procfwk].[ExecutionWrapper]",
"queryTimeout": "02:00:00"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
}
}
},
{
"name": "Archive Execution Log",
"description": "After a successful execution run the current execution metadata is moved to the long term logging table by this stored procedure call.",
"type": "SqlServerStoredProcedure",
"dependsOn": [
{
"activity": "Execute Stages",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[[procfwk].[UpdateExecutionLog]"
},
"linkedServiceName": {
"referenceName": "SupportDatabase",
"type": "LinkedServiceReference"
}
},
{
"name": "Get Tenant Id",
"description": "Returning the Azure Tenant Id from the metadata properties table.",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]",
"storedProcedureParameters": {
"PropertyName": {
"type": "String",
"value": "TenantId"
}
},
"queryTimeout": "02:00:00"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
},
"firstRowOnly": true
}
},
{
"name": "Get Subscription Id",
"description": "Returning the Azure Subscription Id from the metadata properties table.",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": true
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]",
"storedProcedureParameters": {
"PropertyName": {
"type": "String",
"value": "SubscriptionId"
}
},
"queryTimeout": "02:00:00"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
}
}
}
],
"folder": {
"name": "_ProcFwk"
},
"annotations": [
"ADF.procfwk"
]
},
"dependsOn": [
"[concat(variables('factoryId'), '/datasets/GetSetMetadata')]",
"[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]",
"[concat(variables('factoryId'), '/pipelines/03 Child')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/03 Child')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"description": "ADF.procfwk child pipeline used to execute processing pipelines within a given execution stage. This pipeline will be called multi times in parallel.",
"activities": [
{
"name": "Get Pipelines",
"description": "Returns all pipelines from the metadata to be executed within a given processing stage.",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[[procfwk].[GetPipelinesInStage]",
"storedProcedureParameters": {
"StageId": {
"type": "Int32",
"value": {
"value": "@pipeline().parameters.StageId",
"type": "Expression"
}
}
},
"queryTimeout": "02:00:00"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
},
"firstRowOnly": false
}
},
{
"name": "Execute Pipelines",
"description": "Second level ForEach to run in parallel all pipelines within the stage. Items for iteration passed from the Get Pipelines lookup activity.",
"type": "ForEach",
"dependsOn": [
{
"activity": "Get Pipelines",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@activity('Get Pipelines').output.value",
"type": "Expression"
},
"activities": [
{
"name": "Execute Pipeline",
"description": "The lowest level executor with the metadata framework to call existing processing pipelines within Data Factory. The function called will block processing and wait for an outcome.",
"type": "AzureFunctionActivity",
"dependsOn": [
{
"activity": "Log Pipeline Running",
"dependencyConditions": [
"Succeeded"
]
},
{
"activity": "Get Pipeline Params",
"dependencyConditions": [
"Succeeded"
]
},
{
"activity": "Get SPN Details",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"functionName": "ExecutePipeline",
"method": "POST",
"headers": {},
"body": {
"value": "@concat('\n{\n\t\"tenantId\": \"',pipeline().parameters.TenantId,'\",\n\t\"applicationId\": \"',activity('Get SPN Details').output.firstRow.Id,'\",\n\t\"authenticationKey\": \"',activity('Get SPN Details').output.firstRow.Secret,'\",\n\t\"subscriptionId\": \"',pipeline().parameters.SubscriptionId,'\",\n\t\"resourceGroup\": \"',item().ResourceGroupName,'\",\n\t\"factoryName\": \"',item().DataFactoryName,'\",\n\t\"pipelineName\": \"',item().PipelineName,'\"',activity('Get Pipeline Params').output.firstRow.Params,'\n}')",
"type": "Expression"
}
},
"linkedServiceName": {
"referenceName": "PipelineExecutor",
"type": "LinkedServiceReference"
}
},
{
"name": "Get Pipeline Params",
"description": "Returns any parameters from metadata required for the processing pipeline being called. The output can be an empty string if no parameters are required.",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[[procfwk].[GetPipelineParameters]",
"storedProcedureParameters": {
"PipelineId": {
"type": "Int32",
"value": {
"value": "@item().PipelineId",
"type": "Expression"
}
}
},
"queryTimeout": "02:00:00"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
}
}
},
{
"name": "Log Pipeline Running",
"description": "Sets the current pipeline with a status of running within the current execution database table.",
"type": "SqlServerStoredProcedure",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[[procfwk].[SetLogPipelineRunning]",
"storedProcedureParameters": {
"ExecutionId": {
"value": {
"value": "@pipeline().parameters.ExecutionId",
"type": "Expression"
},
"type": "Guid"
},
"PipelineId": {
"value": {
"value": "@item().PipelineId",
"type": "Expression"
},
"type": "Int32"
},
"StageId": {
"value": {
"value": "@pipeline().parameters.StageId",
"type": "Expression"
},
"type": "Int32"
}
}
},
"linkedServiceName": {
"referenceName": "SupportDatabase",
"type": "LinkedServiceReference"
}
},
{
"name": "Get SPN Details",
"description": "Return the SPN ID and Secret for the processing pipeline being executed. Called at this level as each pipeline can have a different SPN.",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[[procfwk].[GetServicePrincipal]",
"storedProcedureParameters": {
"DataFactory": {
"type": "String",
"value": {
"value": "@item().DataFactoryName",
"type": "Expression"
}
},
"PipelineName": {
"type": "String",
"value": {
"value": "@item().PipelineName",
"type": "Expression"
}
}
},
"queryTimeout": "02:00:00"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
}
}
},
{
"name": "Handle Function Output",
"description": "Receives the outcome from the function execution for a given processing pipeline and updates the current execution table with different pipelines status values depending on the result (case).",
"type": "Switch",
"dependsOn": [
{
"activity": "Execute Pipeline",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"on": {
"value": "@activity('Execute Pipeline').output.Status",
"type": "Expression"
},
"cases": [
{
"value": "Failed",
"activities": [
{
"name": "Log Pipeline Failure",
"description": "Updates the current execution table with a pipeline status of failed if the function outcome is failed. Also blocks pipelines in the downstream execution stage.",
"type": "SqlServerStoredProcedure",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[[procfwk].[SetLogPipelineFailed]",
"storedProcedureParameters": {
"ExecutionId": {
"value": {
"value": "@pipeline().parameters.ExecutionId",
"type": "Expression"
},
"type": "Guid"
},
"PipelineId": {
"value": {
"value": "@item().PipelineId",
"type": "Expression"
},
"type": "Int32"
},
"RunId": {
"value": {
"value": "@activity('Execute Pipeline').output.RunIdUsed",
"type": "Expression"
},
"type": "Guid"
},
"StageId": {
"value": {
"value": "@pipeline().parameters.StageId",
"type": "Expression"
},
"type": "Int32"
}
}
},
"linkedServiceName": {
"referenceName": "SupportDatabase",
"type": "LinkedServiceReference"
}
}
]
},
{
"value": "Succeeded",
"activities": [
{
"name": "Log Pipeline Success",
"description": "Updates the current execution table with a pipeline status of success if the function outcome is succeeded.",
"type": "SqlServerStoredProcedure",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]",
"storedProcedureParameters": {
"ExecutionId": {
"value": {
"value": "@pipeline().parameters.ExecutionId",
"type": "Expression"
},
"type": "Guid"
},
"PipelineId": {
"value": {
"value": "@item().PipelineId\n",
"type": "Expression"
},
"type": "Int32"
},
"StageId": {
"value": {
"value": "@pipeline().parameters.StageId",
"type": "Expression"
},
"type": "Int32"
}
}
},
"linkedServiceName": {
"referenceName": "SupportDatabase",
"type": "LinkedServiceReference"
}
}
]
}
],
"defaultActivities": [
{
"name": "Log Pipeline Unknown",
"description": "Updates the current execution table with a pipeline status of unknown if the function returns an unexpected outcome.",
"type": "SqlServerStoredProcedure",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[[procfwk].[SetLogPipelineUnknown]",
"storedProcedureParameters": {
"ExecutionId": {
"value": {
"value": "@pipeline().parameters.ExecutionId",
"type": "Expression"
},
"type": "Guid"
},
"PipelineId": {
"value": {
"value": "@item().PipelineId",
"type": "Expression"
},
"type": "Int32"
},
"StageId": {
"value": {
"value": "@pipeline().parameters.StageId",
"type": "Expression"
},
"type": "Int32"
}
}
},
"linkedServiceName": {
"referenceName": "SupportDatabase",
"type": "LinkedServiceReference"
}
}
]
}
},
{
"name": "Log Activity Failure",
"description": "Handle true failures from calling out to the Azure Function and update the current execution table accordingly so a restart can occur.",
"type": "SqlServerStoredProcedure",
"dependsOn": [
{
"activity": "Execute Pipeline",
"dependencyConditions": [
"Failed"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[[procfwk].[SetLogActivityFailed]",
"storedProcedureParameters": {
"ExecutionId": {
"value": {
"value": "@pipeline().parameters.ExecutionId",
"type": "Expression"
},
"type": "Guid"
},
"PipelineId": {
"value": {
"value": "@item().PipelineId",
"type": "Expression"
},
"type": "Int32"
},
"RunId": {
"value": {
"value": "@pipeline().RunId",
"type": "Expression"
},
"type": "Guid"
},
"StageId": {
"value": {
"value": "@pipeline().parameters.StageId",
"type": "Expression"
},
"type": "Int32"
}
}
},
"linkedServiceName": {
"referenceName": "SupportDatabase",
"type": "LinkedServiceReference"
}
}
]
}
}
],
"parameters": {
"StageId": {
"type": "int"
},
"ExecutionId": {
"type": "string"
},
"TenantId": {
"type": "string"
},
"SubscriptionId": {
"type": "string"
}
},
"variables": {
"FunctionBody": {
"type": "String"
}
},
"folder": {
"name": "_ProcFwk"
},
"annotations": [
"ADF.procfwk"
]
},
"dependsOn": [
"[concat(variables('factoryId'), '/datasets/GetSetMetadata')]",
"[concat(variables('factoryId'), '/linkedServices/PipelineExecutor')]",
"[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/Intention Error')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"description": "Used just so the ADF.procfwk has something to call during development.",
"activities": [
{
"name": "Call Fail Procedure",
"type": "SqlServerStoredProcedure",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[[dbo].[FailProcedure]"
},
"linkedServiceName": {
"referenceName": "SupportDatabase",
"type": "LinkedServiceReference"
}
}
],
"folder": {
"name": "Process Pipelines"
},
"annotations": []
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/Wait 1')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"description": "Used just so the ADF.procfwk has something to call during development.",
"activities": [
{
"name": "Wait1",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": {
"value": "@pipeline().parameters.WaitTime",
"type": "Expression"
}
}
}
],
"parameters": {
"WaitTime": {
"type": "int",
"defaultValue": 5
}
},
"folder": {
"name": "Process Pipelines"
},
"annotations": []
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/Wait 10')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"description": "Used just so the ADF.procfwk has something to call during development.",
"activities": [
{
"name": "Wait10",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": {
"value": "@pipeline().parameters.WaitTime",
"type": "Expression"
}
}
}
],
"parameters": {
"WaitTime": {
"type": "int",
"defaultValue": 5
}
},
"folder": {
"name": "Process Pipelines"
},
"annotations": []
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/Wait 2')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"description": "Used just so the ADF.procfwk has something to call during development.",
"activities": [
{
"name": "Wait2",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": {
"value": "@pipeline().parameters.WaitTime",
"type": "Expression"
}
}
}
],
"parameters": {
"WaitTime": {
"type": "int",
"defaultValue": 5
}
},
"folder": {
"name": "Process Pipelines"
},
"annotations": []
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/Wait 3')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"description": "Used just so the ADF.procfwk has something to call during development.",
"activities": [
{
"name": "Wait3",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": {
"value": "@pipeline().parameters.WaitTime",
"type": "Expression"
}
}
}
],
"parameters": {
"WaitTime": {
"type": "int",
"defaultValue": 5
}
},
"folder": {
"name": "Process Pipelines"
},
"annotations": []
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/Wait 4')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"description": "Used just so the ADF.procfwk has something to call during development.",
"activities": [
{
"name": "Wait4",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": {
"value": "@pipeline().parameters.WaitTime",
"type": "Expression"
}
}
}
],
"parameters": {
"WaitTime": {
"type": "int",
"defaultValue": 5
}
},
"folder": {
"name": "Process Pipelines"
},
"annotations": []
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/Wait 5')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"description": "Used just so the ADF.procfwk has something to call during development.",
"activities": [
{
"name": "Wait5",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": {
"value": "@pipeline().parameters.WaitTime",
"type": "Expression"
}
}
}
],
"parameters": {
"WaitTime": {
"type": "int",
"defaultValue": 5
}
},
"folder": {
"name": "Process Pipelines"
},
"annotations": []
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/Wait 6')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"description": "Used just so the ADF.procfwk has something to call during development.",
"activities": [
{
"name": "Wait6",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": {
"value": "@pipeline().parameters.WaitTime",
"type": "Expression"
}
}
}
],
"parameters": {
"WaitTime": {
"type": "int",
"defaultValue": 5
}
},
"folder": {
"name": "Process Pipelines"
},
"annotations": []
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/Wait 7')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"description": "Used just so the ADF.procfwk has something to call during development.",
"activities": [
{
"name": "Wait7",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": {
"value": "@pipeline().parameters.WaitTime",
"type": "Expression"
}
}
}
],
"parameters": {
"WaitTime": {
"type": "int",
"defaultValue": 5
}
},
"folder": {
"name": "Process Pipelines"
},
"annotations": []
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/Wait 8')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"description": "Used just so the ADF.procfwk has something to call during development.",
"activities": [
{
"name": "Wait8",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": {
"value": "@pipeline().parameters.WaitTime",
"type": "Expression"
}
}
}
],
"parameters": {
"WaitTime": {
"type": "int",
"defaultValue": 5
}
},
"folder": {
"name": "Process Pipelines"
},
"annotations": []
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/Wait 9')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"description": "Used just so the ADF.procfwk has something to call during development.",
"activities": [
{
"name": "Wait9",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": {
"value": "@pipeline().parameters.WaitTime",
"type": "Expression"
}
}
}
],
"parameters": {
"WaitTime": {
"type": "int",
"defaultValue": 5
}
},
"folder": {
"name": "Process Pipelines"
},
"annotations": []
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/GetSetMetadata')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "SupportDatabase",
"type": "LinkedServiceReference"
},
"annotations": [
"ADF.procfwk"
],
"type": "AzureSqlTable",
"schema": [],
"typeProperties": {}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/Keys')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"description": "Connection to Key Vault for all other ADF linked service credentials required to run the processing framework.",
"annotations": [
"ADF.procfwk"
],
"type": "AzureKeyVault",
"typeProperties": {
"baseUrl": "[parameters('Keys_properties_typeProperties_baseUrl')]"
}
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/PipelineExecutor')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"description": "Connection to the Function App from ADF for calling the processing pipeline function within the orchestration framework.",
"annotations": [
"ADF.procfwk"
],
"type": "AzureFunction",
"typeProperties": {
"functionAppUrl": "https://pipelineexecutor.azurewebsites.net",
"functionKey": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "Keys",
"type": "LinkedServiceReference"
},
"secretName": "ExecutorFunctionAppKey"
}
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/Keys')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/SupportDatabase')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"description": "Connection between ADF and processing framework metadata SQLDB.",
"annotations": [
"ADF.procfwk"
],
"type": "AzureSqlDatabase",
"typeProperties": {
"connectionString": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "Keys",
"type": "LinkedServiceReference"
},
"secretName": "[parameters('SupportDatabase_properties_typeProperties_connectionString_secretName')]"
}
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/Keys')]"
]
}
]
}
================================================
FILE: ARM Templates/Data Factory/v1.3 Export.json
================================================
{
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"factoryName": {
"type": "string",
"metadata": "Data Factory name",
"defaultValue": ""
},
"Keys_properties_typeProperties_baseUrl": {
"type": "string",
"defaultValue": ""
},
"SupportDatabase_properties_typeProperties_connectionString_secretName": {
"type": "string",
"defaultValue": ""
}
},
"variables": {
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
},
"resources": [
{
"name": "[concat(parameters('factoryName'), '/01-Grandparent')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"description": "ADF.procfwk grandparent pipeline used optionally to bootstrap any wider processes in your Data Factory that then calls the processing framework.",
"activities": [
{
"name": "Framework Processing",
"type": "ExecutePipeline",
"dependsOn": [
{
"activity": "Set Random Waits",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"pipeline": {
"referenceName": "02-Parent",
"type": "PipelineReference"
},
"waitOnCompletion": true,
"parameters": {}
}
},
{
"name": "Set Random Waits",
"description": "For functional testing only.",
"type": "SqlServerStoredProcedure",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[[dbo].[SetRandomWaitValues]"
},
"linkedServiceName": {
"referenceName": "SupportDatabase",
"type": "LinkedServiceReference"
}
}
],
"folder": {
"name": "_ProcFwk"
},
"annotations": [
"ADF.procfwk"
]
},
"dependsOn": [
"[concat(variables('factoryId'), '/pipelines/02-Parent')]",
"[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/02-Parent')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"description": "ADF.procfwk parent pipeline used to bootstrap the orchestration framework in perform the first level ForEach calls in sequence for the metadata stages.",
"activities": [
{
"name": "Get Stages",
"description": "Returns a distinct list of execution stages within the framework metadata.",
"type": "Lookup",
"dependsOn": [
{
"activity": "Execution Wrapper",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[[procfwk].[GetStages]",
"storedProcedureParameters": {
"ExecutionId": {
"type": "Guid",
"value": {
"value": "@activity('Execution Wrapper').output.firstRow.ExecutionId",
"type": "Expression"
}
}
},
"queryTimeout": "02:00:00"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
},
"firstRowOnly": false
}
},
{
"name": "Execute Stages",
"description": "Top level ForEach to sequentially call all processing stages within the framework metadata. Items for iteration passed from the Get Stages lookup activity.",
"type": "ForEach",
"dependsOn": [
{
"activity": "Get Stages",
"dependencyConditions": [
"Succeeded"
]
},
{
"activity": "Get Tenant Id",
"dependencyConditions": [
"Succeeded"
]
},
{
"activity": "Get Subscription Id",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@activity('Get Stages').output.value",
"type": "Expression"
},
"isSequential": true,
"activities": [
{
"name": "Stage Executor",
"description": "Call to the framework generic child pipeline for a given execution stage.",
"type": "ExecutePipeline",
"dependsOn": [
{
"activity": "Log Stage Preparing",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"pipeline": {
"referenceName": "03-Child",
"type": "PipelineReference"
},
"waitOnCompletion": true,
"parameters": {
"StageId": {
"value": "@item().StageId",
"type": "Expression"
},
"ExecutionId": {
"value": "@activity('Execution Wrapper').output.firstRow.ExecutionId",
"type": "Expression"
},
"TenantId": {
"value": "@activity('Get Tenant Id').output.firstRow.PropertyValue",
"type": "Expression"
},
"SubscriptionId": {
"value": "@activity('Get Subscription Id').output.firstRow.PropertyValue",
"type": "Expression"
}
}
}
},
{
"name": "Log Stage Preparing",
"description": "Update the current execution table flagging all pipelines within the stage as preparing.",
"type": "SqlServerStoredProcedure",
"dependsOn": [
{
"activity": "Check for Blockers",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[[procfwk].[SetLogStagePreparing]",
"storedProcedureParameters": {
"ExecutionId": {
"value": {
"value": "@activity('Execution Wrapper').output.firstRow.ExecutionId",
"type": "Expression"
},
"type": "Guid"
},
"StageId": {
"value": {
"value": "@item().StageId",
"type": "Expression"
},
"type": "Int32"
}
}
},
"linkedServiceName": {
"referenceName": "SupportDatabase",
"type": "LinkedServiceReference"
}
},
{
"name": "Check for Blockers",
"description": "Used to double check and stop the next execution stage if failures and blockers have be incurred. Without this step processing would continue regardless of upstream failures.",
"type": "SqlServerStoredProcedure",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[[procfwk].[CheckForBlockedPipelines]",
"storedProcedureParameters": {
"StageId": {
"value": {
"value": "@item().StageId",
"type": "Expression"
},
"type": "Int32"
}
}
},
"linkedServiceName": {
"referenceName": "SupportDatabase",
"type": "LinkedServiceReference"
}
}
]
}
},
{
"name": "Execution Wrapper",
"description": "Wrapper to reset and restart processing or create a completely new execution instance of the framework metadata.",
"type": "Lookup",
"dependsOn": [
{
"activity": "Metadata Integrity Checks",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[[procfwk].[ExecutionWrapper]",
"storedProcedureParameters": {
"CallingDataFactory": {
"type": "String",
"value": {
"value": "@pipeline().DataFactory",
"type": "Expression"
}
}
},
"queryTimeout": "02:00:00"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
}
}
},
{
"name": "Archive Execution Log",
"description": "After a successful execution run the current execution metadata is moved to the long term logging table by this stored procedure call.",
"type": "SqlServerStoredProcedure",
"dependsOn": [
{
"activity": "Execute Stages",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[[procfwk].[UpdateExecutionLog]"
},
"linkedServiceName": {
"referenceName": "SupportDatabase",
"type": "LinkedServiceReference"
}
},
{
"name": "Get Tenant Id",
"description": "Returning the Azure Tenant Id from the metadata properties table.",
"type": "Lookup",
"dependsOn": [
{
"activity": "Metadata Integrity Checks",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]",
"storedProcedureParameters": {
"PropertyName": {
"type": "String",
"value": "TenantId"
}
},
"queryTimeout": "02:00:00"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
},
"firstRowOnly": true
}
},
{
"name": "Get Subscription Id",
"description": "Returning the Azure Subscription Id from the metadata properties table.",
"type": "Lookup",
"dependsOn": [
{
"activity": "Metadata Integrity Checks",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": true
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]",
"storedProcedureParameters": {
"PropertyName": {
"type": "String",
"value": "SubscriptionId"
}
},
"queryTimeout": "02:00:00"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
}
}
},
{
"name": "Metadata Integrity Checks",
"description": "Performs a series of checks on all metadata held in the framework SQLDB. This is intended to raise errors before an execution run even starts.",
"type": "SqlServerStoredProcedure",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[[procfwk].[CheckMetadataIntegrity]",
"storedProcedureParameters": {
"DebugMode": {
"value": "false",
"type": "Boolean"
}
}
},
"linkedServiceName": {
"referenceName": "SupportDatabase",
"type": "LinkedServiceReference"
}
}
],
"folder": {
"name": "_ProcFwk"
},
"annotations": [
"ADF.procfwk"
]
},
"dependsOn": [
"[concat(variables('factoryId'), '/datasets/GetSetMetadata')]",
"[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]",
"[concat(variables('factoryId'), '/pipelines/03-Child')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/03-Child')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"description": "ADF.procfwk child pipeline used to execute processing pipelines within a given execution stage. This pipeline will be called multi times in parallel.",
"activities": [
{
"name": "Get Pipelines",
"description": "Returns all pipelines from the metadata to be executed within a given processing stage.",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[[procfwk].[GetPipelinesInStage]",
"storedProcedureParameters": {
"StageId": {
"type": "Int32",
"value": {
"value": "@pipeline().parameters.StageId",
"type": "Expression"
}
}
},
"queryTimeout": "02:00:00"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
},
"firstRowOnly": false
}
},
{
"name": "Execute Pipelines",
"description": "Second level ForEach to run in parallel all pipelines within the stage. Items for iteration passed from the Get Pipelines lookup activity.",
"type": "ForEach",
"dependsOn": [
{
"activity": "Get Pipelines",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@activity('Get Pipelines').output.value",
"type": "Expression"
},
"activities": [
{
"name": "Execute Pipeline",
"description": "The lowest level executor with the metadata framework to call existing processing pipelines within Data Factory. The function called will block processing and wait for an outcome.",
"type": "AzureFunctionActivity",
"dependsOn": [
{
"activity": "Log Pipeline Running",
"dependencyConditions": [
"Succeeded"
]
},
{
"activity": "Get Pipeline Params",
"dependencyConditions": [
"Succeeded"
]
},
{
"activity": "Get SPN Details",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"functionName": "ExecutePipeline",
"method": "POST",
"headers": {},
"body": {
"value": "@concat('\n{\n\t\"tenantId\": \"',pipeline().parameters.TenantId,'\",\n\t\"applicationId\": \"',activity('Get SPN Details').output.firstRow.Id,'\",\n\t\"authenticationKey\": \"',activity('Get SPN Details').output.firstRow.Secret,'\",\n\t\"subscriptionId\": \"',pipeline().parameters.SubscriptionId,'\",\n\t\"resourceGroup\": \"',item().ResourceGroupName,'\",\n\t\"factoryName\": \"',item().DataFactoryName,'\",\n\t\"pipelineName\": \"',item().PipelineName,'\"',activity('Get Pipeline Params').output.firstRow.Params,'\n}')",
"type": "Expression"
}
},
"linkedServiceName": {
"referenceName": "PipelineExecutor",
"type": "LinkedServiceReference"
}
},
{
"name": "Get Pipeline Params",
"description": "Returns any parameters from metadata required for the processing pipeline being called. The output can be an empty string if no parameters are required.",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[[procfwk].[GetPipelineParameters]",
"storedProcedureParameters": {
"PipelineId": {
"type": "Int32",
"value": {
"value": "@item().PipelineId",
"type": "Expression"
}
}
},
"queryTimeout": "02:00:00"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
}
}
},
{
"name": "Log Pipeline Running",
"description": "Sets the current pipeline with a status of running within the current execution database table.",
"type": "SqlServerStoredProcedure",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[[procfwk].[SetLogPipelineRunning]",
"storedProcedureParameters": {
"ExecutionId": {
"value": {
"value": "@pipeline().parameters.ExecutionId",
"type": "Expression"
},
"type": "Guid"
},
"PipelineId": {
"value": {
"value": "@item().PipelineId",
"type": "Expression"
},
"type": "Int32"
},
"StageId": {
"value": {
"value": "@pipeline().parameters.StageId",
"type": "Expression"
},
"type": "Int32"
}
}
},
"linkedServiceName": {
"referenceName": "SupportDatabase",
"type": "LinkedServiceReference"
}
},
{
"name": "Get SPN Details",
"description": "Return the SPN ID and Secret for the processing pipeline being executed. Called at this level as each pipeline can have a different SPN.",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[[procfwk].[GetServicePrincipal]",
"storedProcedureParameters": {
"DataFactory": {
"type": "String",
"value": {
"value": "@item().DataFactoryName",
"type": "Expression"
}
},
"PipelineName": {
"type": "String",
"value": {
"value": "@item().PipelineName",
"type": "Expression"
}
}
},
"queryTimeout": "02:00:00"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
}
}
},
{
"name": "Handle Function Output",
"description": "Receives the outcome from the function execution for a given processing pipeline and updates the current execution table with different pipelines status values depending on the result (case).",
"type": "Switch",
"dependsOn": [
{
"activity": "Execute Pipeline",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"on": {
"value": "@activity('Execute Pipeline').output.Status",
"type": "Expression"
},
gitextract_8_ebrzfj/
├── .gitattributes
├── .github/
│ ├── FUNDING.yml
│ └── ISSUE_TEMPLATE/
│ ├── bug-found.md
│ ├── feature-request.md
│ └── help---support-request.md
├── .gitignore
├── .vscode/
│ ├── extensions.json
│ ├── launch.json
│ ├── settings.json
│ └── tasks.json
├── ARM Templates/
│ ├── Data Factory/
│ │ ├── v1.0 Export.json
│ │ ├── v1.1 Export.json
│ │ ├── v1.2 Export.json
│ │ ├── v1.3 Export.json
│ │ ├── v1.4 Export.json
│ │ ├── v1.5 Export.json
│ │ ├── v1.6 Export.json
│ │ ├── v1.7 Export.json
│ │ ├── v1.8 Export.json
│ │ ├── v1.8.3 Export.json
│ │ ├── v1.8.5 Export.json
│ │ ├── v1.8.6 Export.json
│ │ ├── v1.9 Export.json
│ │ ├── v1.9.1 Export.json
│ │ ├── v1.9.2 Export.json
│ │ └── v2.0 Export.json
│ ├── Functions App/
│ │ └── v1.6 Export.json
│ ├── SQL Database/
│ │ └── v1.6 Export.json
│ └── Synapse/
│ └── v2.0 Export.json
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── DataFactory/
│ ├── dataset/
│ │ └── GetSetMetadata.json
│ ├── factory/
│ │ └── FrameworkFactory.json
│ ├── integrationRuntime/
│ │ └── AzureIR-UKSouth.json
│ ├── linkedService/
│ │ ├── FrameworkFunctions.json
│ │ ├── Keys.json
│ │ └── SupportDatabase.json
│ ├── pipeline/
│ │ ├── 01-Grandparent.json
│ │ ├── 02-Parent.json
│ │ ├── 03-Child.json
│ │ ├── 04-Infant.json
│ │ ├── Check For Running Pipeline.json
│ │ ├── Email Sender.json
│ │ ├── Intentional Error.json
│ │ ├── Throw Exception.json
│ │ ├── Wait 1.json
│ │ ├── Wait 10.json
│ │ ├── Wait 2.json
│ │ ├── Wait 3.json
│ │ ├── Wait 4.json
│ │ ├── Wait 5.json
│ │ ├── Wait 6.json
│ │ ├── Wait 7.json
│ │ ├── Wait 8.json
│ │ └── Wait 9.json
│ └── trigger/
│ └── FunctionalTestingTrigger.json
├── DeploymentTools/
│ ├── DataFactory/
│ │ ├── DeployProcFwkComponents.ps1
│ │ ├── DeployProcFwkComponents_Old.ps1
│ │ ├── Get Pipelines.ps1
│ │ ├── GlobalVars.ps1
│ │ ├── PopulatePipelinesInDb.ps1
│ │ ├── ProcFwkComponents.json
│ │ └── config-all.csv
│ ├── Deployment.targets
│ └── DeploymentTools.deployproj
├── FactoryTesting/
│ ├── FactoryTesting.csproj
│ ├── Helpers/
│ │ ├── CoverageHelper.cs
│ │ ├── DataFactoryHelper.cs
│ │ ├── DatabaseHelper.cs
│ │ ├── PipelineRunHelper.cs
│ │ └── SettingsHelper.cs
│ ├── Pipelines/
│ │ ├── 01-Grandparent/
│ │ │ ├── Given300WorkerPipelines.cs
│ │ │ ├── Given600WorkerPipelineConcurrentBatches.cs
│ │ │ ├── GivenOneWorkerPipeline.cs
│ │ │ └── GrandparentHelper.cs
│ │ ├── 02-Parent/
│ │ │ ├── Given20ConcurrentBatchesFor1000WorkerPipelines.cs
│ │ │ ├── GivenAlreadyRunning.cs
│ │ │ ├── GivenBatchExecutionsForConcurrentBatches.cs
│ │ │ ├── GivenBatchExecutionsForConcurrentBatchesAlreadyRunning.cs
│ │ │ ├── GivenBatchExecutionsForConcurrentBatchesWithSimpleFailureHandling.cs
│ │ │ ├── GivenBatchExecutionsForConcurrentBatchesWithSimpleFailureHandlingAndRestart.cs
│ │ │ ├── GivenBatchExecutionsForConcurrentBatchesWithSimpleFailureHandlingAndRestartOveride.cs
│ │ │ ├── GivenBatchExecutionsForSingleBatch.cs
│ │ │ ├── GivenCancelledWorkerAndRestart.cs
│ │ │ ├── GivenCancelledWorkerInOneExecutionStage.cs
│ │ │ ├── GivenCancelledWorkerThatBlocks.cs
│ │ │ ├── GivenCancelledWorkerThatDoesntBlock.cs
│ │ │ ├── GivenCleanUpForCancelledWorkerAndRestart.cs
│ │ │ ├── GivenCleanUpForSuccessfulWorkersAndRestart.cs
│ │ │ ├── GivenDependencyChainFailureHandling.cs
│ │ │ ├── GivenDependencyChainFailureHandlingAndRestart.cs
│ │ │ ├── GivenDisabledBatches.cs
│ │ │ ├── GivenDisabledPipelines.cs
│ │ │ ├── GivenDisabledStages.cs
│ │ │ ├── GivenNoErrorsAndSPNStoredInDatabase.cs
│ │ │ ├── GivenNoErrorsAndSPNStoredInKeyVault.cs
│ │ │ ├── GivenNoFailureHandling.cs
│ │ │ ├── GivenNoPipelineParameters.cs
│ │ │ ├── GivenOneExecutionStage.cs
│ │ │ ├── GivenSimpleFailureHandling.cs
│ │ │ ├── GivenSimpleFailureHandlingAndRestart.cs
│ │ │ ├── GivenSimpleFailureHandlingAndRestartOveride.cs
│ │ │ └── ParentHelper.cs
│ │ └── Utilities/
│ │ ├── GivenCheckForRunningPipeline.cs
│ │ ├── GivenEmailSender.cs
│ │ ├── GivenThrowException.cs
│ │ └── UtilitiesHelper.cs
│ ├── dev.runsettings
│ ├── multi.runsettings
│ └── test.runsettings
├── Functions/
│ ├── .vscode/
│ │ ├── extensions.json
│ │ ├── launch.json
│ │ ├── settings.json
│ │ └── tasks.json
│ ├── Functions/
│ │ ├── CancelPipeline.cs
│ │ ├── CheckPipelineStatus.cs
│ │ ├── ExecutePipeline.cs
│ │ ├── GetActivityErrors.cs
│ │ ├── SendEmail.cs
│ │ └── ValidatePipeline.cs
│ ├── Functions.csproj
│ ├── Helpers/
│ │ ├── BodyReader.cs
│ │ ├── InvalidRequestException.cs
│ │ ├── KeyVaultClient.cs
│ │ ├── PipelineRequest.cs
│ │ ├── PipelineRunRequest.cs
│ │ └── SMTPClient.cs
│ ├── Properties/
│ │ └── ServiceDependencies/
│ │ ├── FrameworkSupportFunctions - Zip Deploy/
│ │ │ └── profile.arm.json
│ │ └── FrameworkSupportFunctionsBig - Web Deploy/
│ │ └── profile.arm.json
│ ├── Services/
│ │ ├── AzureDataFactoryService.cs
│ │ ├── AzureSynapseService.cs
│ │ ├── PipelineService.cs
│ │ ├── PipelineServiceType.cs
│ │ └── Returns/
│ │ ├── PipelineDescription.cs
│ │ ├── PipelineErrorDetail.cs
│ │ └── PipelineRunStatus.cs
│ ├── host.json
│ └── template_local.settings.json
├── Images/
│ └── procfwk Designs.vsdx
├── LICENSE
├── MetadataDB/
│ ├── MetadataDB.refactorlog
│ ├── MetadataDB.sqlproj
│ ├── Scripts/
│ │ ├── Alter Database Scale.sql
│ │ ├── Handy Selects.sql
│ │ ├── LogData/
│ │ │ ├── ErrorLogBackup.sql
│ │ │ ├── ErrorLogRestore.sql
│ │ │ ├── ExecutionLogBackup.sql
│ │ │ └── ExecutionLogRestore.sql
│ │ ├── Metadata/
│ │ │ ├── AlertOutcomes.sql
│ │ │ ├── DeleteAll.sql
│ │ │ ├── DropLegacyObjects.sql
│ │ │ ├── DropLegacyTables.sql
│ │ │ ├── Orchestrators.sql
│ │ │ ├── PipelineDependencies.sql
│ │ │ ├── PipelineParams.sql
│ │ │ ├── Pipelines.sql
│ │ │ ├── Properties.sql
│ │ │ ├── RecipientAlertsLink.sql
│ │ │ ├── Recipients.sql
│ │ │ ├── ReplaceDataFactorys.sql
│ │ │ ├── Stages.sql
│ │ │ ├── TransferHelperObjects.sql
│ │ │ └── TransferReportingObjects.sql
│ │ ├── Script.PostDeployment.sql
│ │ ├── Script.PreDeployment.sql
│ │ └── Script.SetLocalAuthenticationDetails.sql
│ ├── Security/
│ │ ├── procfwk.sql
│ │ ├── procfwkHelpers.sql
│ │ ├── procfwkReporting.sql
│ │ ├── procfwkTesting.sql
│ │ └── procfwkuser Role.sql
│ ├── dbo/
│ │ ├── Stored Procedures/
│ │ │ ├── DemoModePrecursor.sql
│ │ │ ├── ExampleCustomExecutionPrecursor.sql
│ │ │ └── FailProcedure.sql
│ │ └── Tables/
│ │ └── ServicePrincipals.sql
│ ├── procfwk/
│ │ ├── Functions/
│ │ │ └── GetPropertyValueInternal.sql
│ │ ├── Stored Procedures/
│ │ │ ├── BatchWrapper.sql
│ │ │ ├── CheckForBlockedPipelines.sql
│ │ │ ├── CheckForEmailAlerts.sql
│ │ │ ├── CheckMetadataIntegrity.sql
│ │ │ ├── CheckPreviousExeuction.sql
│ │ │ ├── CreateNewExecution.sql
│ │ │ ├── ExecutePrecursorProcedure.sql
│ │ │ ├── ExecutionWrapper.sql
│ │ │ ├── GetEmailAlertParts.sql
│ │ │ ├── GetFrameworkOrchestratorDetails.sql
│ │ │ ├── GetPipelineParameters.sql
│ │ │ ├── GetPipelinesInStage.sql
│ │ │ ├── GetPropertyValue.sql
│ │ │ ├── GetStages.sql
│ │ │ ├── GetWorkerAuthDetails.sql
│ │ │ ├── GetWorkerDetailsWrapper.sql
│ │ │ ├── GetWorkerPipelineDetails.sql
│ │ │ ├── ResetExecution.sql
│ │ │ ├── SetErrorLogDetails.sql
│ │ │ ├── SetExecutionBlockDependants.sql
│ │ │ ├── SetLogActivityFailed.sql
│ │ │ ├── SetLogPipelineCancelled.sql
│ │ │ ├── SetLogPipelineChecking.sql
│ │ │ ├── SetLogPipelineFailed.sql
│ │ │ ├── SetLogPipelineLastStatusCheck.sql
│ │ │ ├── SetLogPipelineRunId.sql
│ │ │ ├── SetLogPipelineRunning.sql
│ │ │ ├── SetLogPipelineSuccess.sql
│ │ │ ├── SetLogPipelineUnknown.sql
│ │ │ ├── SetLogPipelineValidating.sql
│ │ │ ├── SetLogStagePreparing.sql
│ │ │ └── UpdateExecutionLog.sql
│ │ ├── Synonyms/
│ │ │ ├── AddPipelineDependant.sql
│ │ │ ├── AddProperty.sql
│ │ │ ├── AddRecipientPipelineAlerts.sql
│ │ │ ├── AddServicePrincipal.sql
│ │ │ ├── AddServicePrincipalUrls.sql
│ │ │ ├── AddServicePrincipalWrapper.sql
│ │ │ ├── AverageStageDuration.sql
│ │ │ ├── CheckForValidURL.sql
│ │ │ ├── CheckStageAndPiplineIntegrity.sql
│ │ │ ├── CompleteExecutionErrorLog.sql
│ │ │ ├── CompleteExecutionLog.sql
│ │ │ ├── CurrentExecutionSummary.sql
│ │ │ ├── DataFactoryDetails.sql
│ │ │ ├── DeleteRecipientAlerts.sql
│ │ │ ├── DeleteServicePrincipal.sql
│ │ │ ├── GetExecutionDetails.sql
│ │ │ ├── LastExecution.sql
│ │ │ ├── LastExecutionSummary.sql
│ │ │ ├── PipelineDependencyChains.sql
│ │ │ ├── PipelineProcesses.sql
│ │ │ ├── ProcessingStageDetails.sql
│ │ │ └── WorkerParallelismOverTime.sql
│ │ ├── Tables/
│ │ │ ├── AlertOutcomes.sql
│ │ │ ├── BatchExecution.sql
│ │ │ ├── BatchStageLink.sql
│ │ │ ├── Batches.sql
│ │ │ ├── CurrentExecution.sql
│ │ │ ├── ErrorLog.sql
│ │ │ ├── ExecutionLog.sql
│ │ │ ├── Orchestrators.sql
│ │ │ ├── PipelineAlertLink.sql
│ │ │ ├── PipelineAuthLink.sql
│ │ │ ├── PipelineDependencies.sql
│ │ │ ├── PipelineParameters.sql
│ │ │ ├── Pipelines.sql
│ │ │ ├── Properties.sql
│ │ │ ├── Recipients.sql
│ │ │ ├── Stages.sql
│ │ │ ├── Subscriptions.sql
│ │ │ └── Tenants.sql
│ │ └── Views/
│ │ ├── CurrentProperties.sql
│ │ ├── DataFactorys.sql
│ │ └── PipelineParameterDataSizes.sql
│ ├── procfwkHelpers/
│ │ ├── Functions/
│ │ │ └── CheckForValidURL.sql
│ │ ├── Stored Procedures/
│ │ │ ├── AddPipelineDependant.sql
│ │ │ ├── AddPipelineViaPowerShell.sql
│ │ │ ├── AddProperty.sql
│ │ │ ├── AddRecipientPipelineAlerts.sql
│ │ │ ├── AddServicePrincipal.sql
│ │ │ ├── AddServicePrincipalUrls.sql
│ │ │ ├── AddServicePrincipalWrapper.sql
│ │ │ ├── CheckStageAndPiplineIntegrity.sql
│ │ │ ├── DeleteMetadataWithIntegrity.sql
│ │ │ ├── DeleteMetadataWithoutIntegrity.sql
│ │ │ ├── DeleteRecipientAlerts.sql
│ │ │ ├── DeleteServicePrincipal.sql
│ │ │ ├── GetExecutionDetails.sql
│ │ │ ├── GetServicePrincipal.sql
│ │ │ ├── SetDefaultAlertOutcomes.sql
│ │ │ ├── SetDefaultBatchStageLink.sql
│ │ │ ├── SetDefaultBatches.sql
│ │ │ ├── SetDefaultOrchestrators.sql
│ │ │ ├── SetDefaultPipelineDependants.sql
│ │ │ ├── SetDefaultPipelineParameters.sql
│ │ │ ├── SetDefaultPipelines.sql
│ │ │ ├── SetDefaultProperties.sql
│ │ │ ├── SetDefaultRecipientPipelineAlerts.sql
│ │ │ ├── SetDefaultRecipients.sql
│ │ │ ├── SetDefaultStages.sql
│ │ │ ├── SetDefaultSubscription.sql
│ │ │ └── SetDefaultTenant.sql
│ │ └── Views/
│ │ └── PipelineDependencyChains.sql
│ ├── procfwkReporting/
│ │ └── Views/
│ │ ├── AverageStageDuration.sql
│ │ ├── CompleteExecutionErrorLog.sql
│ │ ├── CompleteExecutionLog.sql
│ │ ├── CurrentExecutionSummary.sql
│ │ ├── LastExecution.sql
│ │ ├── LastExecutionSummary.sql
│ │ └── WorkerParallelismOverTime.sql
│ └── procfwkTesting/
│ └── Stored Procedures/
│ ├── Add20BatchesFor1000Workers.sql
│ ├── Add300WorkerPipelineBatches.sql
│ ├── Add300WorkerPipelines.sql
│ ├── CleanUpMetadata.sql
│ ├── GetRunIdWhenAvailable.sql
│ └── ResetMetadata.sql
├── MetadataDBTests/
│ ├── MetadataDBTests.sqlproj
│ ├── Scripts/
│ │ ├── Script.PostDeployment.sql
│ │ └── Script.PreDeployment.sql
│ ├── _TestClasses/
│ │ ├── procfwk_GetPropertyValue.sql
│ │ └── procfwk_GetPropertyValueInternal.sql
│ ├── procfwk_GetPropertyValue/
│ │ ├── setup.sql
│ │ ├── test WHEN property does not exist THEN error raised.sql
│ │ ├── test WHEN property exists THEN property value returned.sql
│ │ ├── test WHEN property invalidated THEN error raised.sql
│ │ └── test WHEN property name is null THEN error raised.sql
│ ├── procfwk_GetPropertyValueInternal/
│ │ ├── test WHEN property does not exist THEN empty string returned.sql
│ │ ├── test WHEN property exists THEN property value returned.sql
│ │ └── test WHEN property name is null THEN empty string returned.sql
│ └── tSQLt.dacpac
├── Notebooks/
│ ├── Databricks - Throw Exception.scala
│ └── Metadata Guide and Handy Code Snippets.ipynb
├── ProcessingFramework.sln
├── README.md
├── Reporting/
│ ├── PowerBI/
│ │ ├── Executions Overview.pbit
│ │ └── Executions Overview.pbix
│ └── Scoping-Template-LDI-Blank.docx
└── Synapse/
├── dataflow/
│ └── MDF _Order_Count.json
├── dataset/
│ └── GetSetMetadata.json
├── factory/
│ └── FrameworkFactory.json
├── integrationRuntime/
│ └── AutoResolveIntegrationRuntime.json
├── linkedService/
│ ├── FrameworkFunctions.json
│ ├── Keys.json
│ ├── SupportDatabase.json
│ ├── covid-tracking.json
│ ├── procfwkforsynapse-WorkspaceDefaultSqlServer.json
│ └── procfwkforsynapse-WorkspaceDefaultStorage.json
├── notebook/
│ └── Getting Started with Delta Lake.json
├── pipeline/
│ ├── 01-Grandparent.json
│ ├── 02-Parent.json
│ ├── 03-Child.json
│ ├── 04-Infant.json
│ ├── Check For Running Pipeline.json
│ ├── Email Sender.json
│ ├── Intentional Error.json
│ ├── Throw Exception.json
│ ├── Wait 1.json
│ ├── Wait 10.json
│ ├── Wait 2.json
│ ├── Wait 3.json
│ ├── Wait 4.json
│ ├── Wait 5.json
│ ├── Wait 6.json
│ ├── Wait 7.json
│ ├── Wait 8.json
│ └── Wait 9.json
└── trigger/
└── FunctionalTestingTrigger.json
SYMBOL INDEX (450 symbols across 89 files)
FILE: FactoryTesting/Helpers/CoverageHelper.cs
class CoverageHelper (line 10) | public class CoverageHelper<T> : DatabaseHelper<T> where T : CoverageHel...
method RunPipeline (line 12) | public new async Task RunPipeline(string pipelineName)
method RunPipelineAndCancel (line 21) | public new async Task RunPipelineAndCancel(string pipelineName)
method RecordActivityRun (line 42) | private void RecordActivityRun(ActivityRun ar, string context)
class CoverageHelperSetup (line 55) | [SetUpFixture]
method SetupCoverageHelper (line 58) | [OneTimeSetUp]
method RecordActivity (line 75) | private void RecordActivity(string pipelineName, Activity act)
FILE: FactoryTesting/Helpers/DataFactoryHelper.cs
class DataFactoryHelper (line 14) | public class DataFactoryHelper<T> : SettingsHelper<T> where T : DataFact...
method InitialiseClient (line 20) | private async Task InitialiseClient()
method TriggerPipeline (line 33) | public async Task<string> TriggerPipeline(string pipelineName, IDictio...
method GetPipelines (line 40) | public async Task<List<PipelineResource>> GetPipelines()
method GetActivityRuns (line 56) | public async Task<List<ActivityRun>> GetActivityRuns(string pipelineRu...
method TearDown (line 73) | public virtual void TearDown()
method GetRunStatus (line 78) | public async Task<string> GetRunStatus(string pipelineRunId)
method GetRunStatus (line 84) | public async Task<string> GetRunStatus(string pipelineRunId, string ad...
method IsInProgress (line 91) | public async Task<bool> IsInProgress(string pipelineRunId)
method IsInProgress (line 97) | public async Task<bool> IsInProgress(string pipelineRunId, string adfN...
method IsQueued (line 104) | public async Task<bool> IsQueued(string pipelineRunId)
method IsCancelling (line 111) | public async Task<bool> IsCancelling(string pipelineRunId)
method IsCancelling (line 118) | public async Task<bool> IsCancelling(string pipelineRunId, string adfN...
method CancelRunningPipeline (line 125) | public async Task<string> CancelRunningPipeline(string pipelineRunId, ...
method CancelRunningPipeline (line 144) | public async Task<string> CancelRunningPipeline(string pipelineRunId, ...
method DataFactoryHelper (line 163) | public DataFactoryHelper()
FILE: FactoryTesting/Helpers/DatabaseHelper.cs
class DatabaseHelper (line 7) | public class DatabaseHelper<T> : PipelineRunHelper<T> where T : Database...
method DatabaseHelper (line 11) | public DatabaseHelper()
method WithEmptyTable (line 17) | public T WithEmptyTable(string tableName)
method RowCount (line 24) | public int RowCount(string tableName, string where = "", string equals...
method ColumnData (line 36) | public string ColumnData(string tableName, string columnName, char sep...
method ExecuteStoredProcedure (line 46) | public void ExecuteStoredProcedure(string spName, Dictionary<string, o...
method ExecuteNonQuery (line 58) | public void ExecuteNonQuery(string sql)
method AddTenantAndSubscription (line 64) | public void AddTenantAndSubscription(string tenantId = null, string su...
method AddWorkerSPNStoredInDatabase (line 74) | public void AddWorkerSPNStoredInDatabase(string workerFactoryName, str...
method AddWorkerSPNStoredInKeyVault (line 90) | public void AddWorkerSPNStoredInKeyVault(string workerFactoryName, str...
method AddBasicMetadata (line 106) | public void AddBasicMetadata()
method TearDown (line 113) | public override void TearDown()
FILE: FactoryTesting/Helpers/PipelineRunHelper.cs
class PipelineRunHelper (line 12) | public class PipelineRunHelper<T> : DataFactoryHelper<T> where T : Pipel...
method PipelineRunHelper (line 22) | public PipelineRunHelper()
method WithParameter (line 30) | public T WithParameter(string name, object value)
method RunPipeline (line 36) | public async Task RunPipeline(string pipelineName)
method RunPipelineAndCancel (line 48) | public async Task RunPipelineAndCancel(string pipelineName)
method GetActivityRuns (line 62) | public async Task<IEnumerable<ActivityRun>> GetActivityRuns()
method GetActivityRunCount (line 68) | public async Task<int> GetActivityRunCount(string pattern = ".*")
method InitialiseActivityRuns (line 75) | private async Task InitialiseActivityRuns()
method GetActivityOutput (line 81) | public async Task<string> GetActivityOutput(string activityName, strin...
FILE: FactoryTesting/Helpers/SettingsHelper.cs
class SettingsHelper (line 8) | public class SettingsHelper<T>
method GetSetting (line 10) | public string GetSetting(string settingName)
method SettingsHelper (line 35) | public SettingsHelper()
FILE: FactoryTesting/Pipelines/01-Grandparent/Given300WorkerPipelines.cs
class Given300WorkerPipelines (line 8) | public class Given300WorkerPipelines
method WhenPipelineRun (line 12) | [OneTimeSetUp]
method ThenPipelineOutcomeIsSucceeded (line 26) | [Test]
method ThenCurrentExecutionTableIsEmpty (line 31) | [Test]
method Then300ExecutionLogRecords (line 37) | [Test]
method TearDown (line 45) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/01-Grandparent/Given600WorkerPipelineConcurrentBatches.cs
class Given600WorkerPipelineConcurrentBatches (line 8) | public class Given600WorkerPipelineConcurrentBatches
method WhenPipelineRun (line 13) | [OneTimeSetUp]
method ThenFirstBatchPipelineOutcomeIsSucceeded (line 36) | [Test]
method ThenSecondBatchPipelineOutcomeIsSucceeded (line 42) | [Test]
method ThenTwoBatchExecutionSuccessRecords (line 48) | [Test]
method ThenCurrentExecutionTableIsEmpty (line 54) | [Test]
method Then300ExecutionLogRecords (line 60) | [Test]
method TearDown (line 68) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/01-Grandparent/GivenOneWorkerPipeline.cs
class GivenOneWorkerPipeline (line 8) | public class GivenOneWorkerPipeline
method WhenPipelineRun (line 12) | [OneTimeSetUp]
method ThenPipelineOutcomeIsSucceeded (line 27) | [Test]
method TearDown (line 35) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/01-Grandparent/GrandparentHelper.cs
class GrandparentHelper (line 7) | class GrandparentHelper : CoverageHelper<GrandparentHelper>
method RunPipeline (line 9) | public async Task RunPipeline()
method WithTenantAndSubscriptionIds (line 14) | public GrandparentHelper WithTenantAndSubscriptionIds()
method WithSPNInDatabase (line 20) | public GrandparentHelper WithSPNInDatabase(string workerFactoryName)
method WithSPNInKeyVault (line 26) | public GrandparentHelper WithSPNInKeyVault(string workerFactoryName)
method WithBasicMetadata (line 32) | public GrandparentHelper WithBasicMetadata()
method WithEmptyExecutionTables (line 38) | public GrandparentHelper WithEmptyExecutionTables()
method WithBatchExecutionHandling (line 47) | public GrandparentHelper WithBatchExecutionHandling()
method WithSimpleFailureHandling (line 55) | public GrandparentHelper WithSimpleFailureHandling()
method WithOneWorkerPipelineEnabled (line 60) | public GrandparentHelper WithOneWorkerPipelineEnabled()
method With300WorkerPipelinesEnabled (line 66) | public GrandparentHelper With300WorkerPipelinesEnabled()
method With20BatchesFor1000WorkersEnabled (line 73) | public GrandparentHelper With20BatchesFor1000WorkersEnabled()
method WithCustom (line 80) | public GrandparentHelper WithCustom()
method With300WorkerPipelineBatches (line 87) | public GrandparentHelper With300WorkerPipelineBatches()
FILE: FactoryTesting/Pipelines/02-Parent/Given20ConcurrentBatchesFor1000WorkerPipelines.cs
class Given20ConcurrentBatchesFor1000WorkerPipelines (line 8) | public class Given20ConcurrentBatchesFor1000WorkerPipelines
method WhenPipelineRun (line 31) | [OneTimeSetUp]
method ThenOneBatchPipelineOutcomeIsSucceeded (line 116) | [Test]
method ThenTwoBatchPipelineOutcomeIsSucceeded (line 122) | [Test]
method ThenElevenBatchPipelineOutcomeIsSucceeded (line 127) | [Test]
method ThenTwelveBatchPipelineOutcomeIsSucceeded (line 132) | [Test]
method ThenTwentyBatchPipelineOutcomeIsSucceeded (line 137) | [Test]
method ThenFifteenBatchPipelineOutcomeIsSucceeded (line 142) | [Test]
method ThenSevenBatchPipelineOutcomeIsSucceeded (line 147) | [Test]
method ThenNineBatchPipelineOutcomeIsSucceeded (line 152) | [Test]
method ThenEightBatchPipelineOutcomeIsSucceeded (line 157) | [Test]
method ThenSixteenBatchPipelineOutcomeIsSucceeded (line 162) | [Test]
method ThenFiveBatchPipelineOutcomeIsSucceeded (line 167) | [Test]
method ThenSixBatchPipelineOutcomeIsSucceeded (line 172) | [Test]
method ThenThirteenBatchPipelineOutcomeIsSucceeded (line 177) | [Test]
method ThenNineteenBatchPipelineOutcomeIsSucceeded (line 182) | [Test]
method ThenFourBatchPipelineOutcomeIsSucceeded (line 187) | [Test]
method ThenEighteenBatchPipelineOutcomeIsSucceeded (line 192) | [Test]
method ThenThreeBatchPipelineOutcomeIsSucceeded (line 197) | [Test]
method ThenFourteenBatchPipelineOutcomeIsSucceeded (line 202) | [Test]
method ThenTenBatchPipelineOutcomeIsSucceeded (line 207) | [Test]
method ThenSeventeenBatchPipelineOutcomeIsSucceeded (line 212) | [Test]
method Then1000ExecutionLogRecords (line 218) | [Test]
FILE: FactoryTesting/Pipelines/02-Parent/GivenAlreadyRunning.cs
class GivenAlreadyRunning (line 8) | public class GivenAlreadyRunning
method WhenPipelineRun (line 13) | [OneTimeSetUp]
method ThenFirstPipelineOutcomeIsSucceeded (line 32) | [Test]
method ThenSecondPipelineOutcomeIsFailed (line 38) | [Test]
method ThenCurrentExecutionTableIsEmpty (line 44) | [Test]
method ThenElevenExecutionLogRecords (line 50) | [Test]
method TearDown (line 56) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/02-Parent/GivenBatchExecutionsForConcurrentBatches.cs
class GivenBatchExecutionsForConcurrentBatches (line 8) | public class GivenBatchExecutionsForConcurrentBatches
method WhenPipelineRun (line 13) | [OneTimeSetUp]
method ThenFirstBatchPipelineOutcomeIsSucceeded (line 36) | [Test]
method ThenSecondBatchPipelineOutcomeIsSucceeded (line 42) | [Test]
method ThenCurrentExecutionTableIsEmpty (line 48) | [Test]
method ThenFifteenExecutionLogSuccessRecords (line 54) | [Test]
method ThenTwoBatchExecutionSuccessRecords (line 60) | [Test]
method TearDown (line 67) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/02-Parent/GivenBatchExecutionsForConcurrentBatchesAlreadyRunning.cs
class GivenBatchExecutionsForConcurrentBatchesAlreadyRunning (line 8) | public class GivenBatchExecutionsForConcurrentBatchesAlreadyRunning
method WhenPipelineRun (line 13) | [OneTimeSetUp]
method ThenFirstBatchPipelineOutcomeIsSucceeded (line 36) | [Test]
method ThenSecondBatchPipelineOutcomeIsFailed (line 42) | [Test]
method ThenCurrentExecutionTableIsEmpty (line 48) | [Test]
method ThenFourExecutionLogSuccessRecords (line 54) | [Test]
method ThenOneBatchExecutionSuccessRecord (line 60) | [Test]
method TearDown (line 67) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/02-Parent/GivenBatchExecutionsForConcurrentBatchesWithSimpleFailureHandling.cs
class GivenBatchExecutionsForConcurrentBatchesWithSimpleFailureHandling (line 8) | public class GivenBatchExecutionsForConcurrentBatchesWithSimpleFailureHa...
method WhenPipelineRun (line 13) | [OneTimeSetUp]
method ThenFirstBatchPipelineOutcomeIsSucceeded (line 39) | [Test]
method ThenOneBatchExecutionSuccessRecord (line 45) | [Test]
method ThenFourExecutionLogSuccessRecords (line 51) | [Test]
method ThenSecondBatchPipelineOutcomeIsFailed (line 58) | [Test]
method ThenOneBatchExecutionStoppedRecord (line 64) | [Test]
method ThenThreeExecutionsSucceeded (line 70) | [Test]
method ThenOneExecutionFailed (line 76) | [Test]
method ThenSevenExecutionsBlocked (line 82) | [Test]
method ThenOneExecutionLogRecord (line 88) | [Test]
method ThenTwoErrorLogRecords (line 94) | [Test]
method TearDown (line 102) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/02-Parent/GivenBatchExecutionsForConcurrentBatchesWithSimpleFailureHandlingAndRestart.cs
class GivenBatchExecutionsForConcurrentBatchesWithSimpleFailureHandlingAndRestart (line 8) | public class GivenBatchExecutionsForConcurrentBatchesWithSimpleFailureHa...
method WhenPipelineRun (line 15) | [OneTimeSetUp]
method ThenFirstBatchPipelineOutcomeIsSucceeded (line 55) | [Test]
method ThenSecondBatchPipelineOutcomeIsFailed (line 62) | [Test]
method ThenSecondBatchRestartPipelineOutcomeIsSucceeded (line 69) | [Test]
method ThenThirdBatchPipelineOutcomeIsSucceeded (line 76) | [Test]
method ThenThreeBatchExecutionSuccessRecords (line 84) | [Test]
method ThenNineteenExecutionLogSuccessRecords (line 90) | [Test]
method ThenOneExecutionLogRecord (line 96) | [Test]
method ThenTwoErrorLogRecords (line 102) | [Test]
method ThenCurrentExecutionTableIsEmpty (line 108) | [Test]
method TearDown (line 116) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/02-Parent/GivenBatchExecutionsForConcurrentBatchesWithSimpleFailureHandlingAndRestartOveride.cs
class GivenBatchExecutionsForConcurrentBatchesWithSimpleFailureHandlingAndRestartOveride (line 8) | public class GivenBatchExecutionsForConcurrentBatchesWithSimpleFailureHa...
method WhenPipelineRun (line 15) | [OneTimeSetUp]
method ThenFirstBatchPipelineOutcomeIsSucceeded (line 56) | [Test]
method ThenSecondBatchPipelineOutcomeIsFailed (line 63) | [Test]
method ThenSecondBatchRestartPipelineOutcomeIsSucceeded (line 70) | [Test]
method ThenThirdBatchPipelineOutcomeIsSucceeded (line 77) | [Test]
method ThenThreeBatchExecutionSuccessRecords (line 85) | [Test]
method ThenOneBatchExecutionAbandonedRecord (line 91) | [Test]
method ThenTwentyTwoExecutionLogSuccessRecords (line 97) | [Test]
method ThenOneExecutionLogRecord (line 103) | [Test]
method ThenSevenExecutionsBlocked (line 108) | [Test]
method ThenTwoErrorLogRecords (line 113) | [Test]
method ThenCurrentExecutionTableIsEmpty (line 119) | [Test]
method TearDown (line 127) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/02-Parent/GivenBatchExecutionsForSingleBatch.cs
class GivenBatchExecutionsForSingleBatch (line 8) | public class GivenBatchExecutionsForSingleBatch
method WhenPipelineRun (line 12) | [OneTimeSetUp]
method ThenPipelineOutcomeIsSucceeded (line 29) | [Test]
method ThenCurrentExecutionTableIsEmpty (line 35) | [Test]
method ThenFourExecutionLogSuccessRecords (line 41) | [Test]
method ThenOneBatchExecutionSuccessRecord (line 47) | [Test]
method TearDown (line 54) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/02-Parent/GivenCancelledWorkerAndRestart.cs
class GivenCancelledWorkerAndRestart (line 8) | public class GivenCancelledWorkerAndRestart
method WhenPipelineRun (line 13) | [OneTimeSetUp]
method ThenPipelineOutcomeIsSucceeded (line 36) | [Test]
method ThenOneExecutionLogRecord (line 42) | [Test]
method ThenElevenExecutionsSucceeded (line 47) | [Test]
method TearDown (line 54) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/02-Parent/GivenCancelledWorkerInOneExecutionStage.cs
class GivenCancelledWorkerInOneExecutionStage (line 8) | public class GivenCancelledWorkerInOneExecutionStage
method WhenPipelineRun (line 12) | [OneTimeSetUp]
method ThenPipelineOutcomeIsFailed (line 32) | [Test]
method ThenOneExecutionsCancelled (line 38) | [Test]
method ThenOneExecutionLogRecord (line 44) | [Test]
method ThenThreeExecutionsSucceeded (line 49) | [Test]
method TearDown (line 56) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/02-Parent/GivenCancelledWorkerThatBlocks.cs
class GivenCancelledWorkerThatBlocks (line 8) | public class GivenCancelledWorkerThatBlocks
method WhenPipelineRun (line 12) | [OneTimeSetUp]
method ThenPipelineOutcomeIsFailed (line 32) | [Test]
method ThenOneExecutionsCancelled (line 38) | [Test]
method ThenOneExecutionLogRecord (line 44) | [Test]
method ThenThreeExecutionsSucceeded (line 49) | [Test]
method ThenSevenExecutionsBlocked (line 54) | [Test]
method TearDown (line 61) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/02-Parent/GivenCancelledWorkerThatDoesntBlock.cs
class GivenCancelledWorkerThatDoesntBlock (line 8) | public class GivenCancelledWorkerThatDoesntBlock
method WhenPipelineRun (line 12) | [OneTimeSetUp]
method ThenPipelineOutcomeIsFailed (line 32) | [Test]
method ThenOneExecutionsCancelled (line 38) | [Test]
method ThenOneExecutionLogRecord (line 44) | [Test]
method ThenTenExecutionsSucceeded (line 49) | [Test]
method TearDown (line 56) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/02-Parent/GivenCleanUpForCancelledWorkerAndRestart.cs
class GivenCleanUpForCancelledWorkerAndRestart (line 8) | public class GivenCleanUpForCancelledWorkerAndRestart
method WhenPipelineRun (line 13) | [OneTimeSetUp]
method ThenPipelineOutcomeIsSucceeded (line 39) | [Test]
method ThenActivityShouldReturnOneRowForCleanUp (line 45) | [Test]
method ThenOneExecutionLogRecord (line 52) | [Test]
method ThenElevenExecutionsSucceeded (line 57) | [Test]
method TearDown (line 64) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/02-Parent/GivenCleanUpForSuccessfulWorkersAndRestart.cs
class GivenCleanUpForSuccessfulWorkersAndRestart (line 8) | public class GivenCleanUpForSuccessfulWorkersAndRestart
method WhenPipelineRun (line 13) | [OneTimeSetUp]
method ThenPipelineOutcomeIsSucceeded (line 38) | [Test]
method ThenActivityShouldReturnThreeRowsForCleanUp (line 44) | [Test]
method ThenOneExecutionLogRecord (line 51) | [Test]
method ThenElevenExecutionsSucceeded (line 56) | [Test]
method TearDown (line 62) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/02-Parent/GivenDependencyChainFailureHandling.cs
class GivenDependencyChainFailureHandling (line 8) | public class GivenDependencyChainFailureHandling
method WhenPipelineRun (line 12) | [OneTimeSetUp]
method ThenPipelineOutcomeIsFailed (line 27) | [Test, Order(1)]
method ThenSixExecutionsSucceeded (line 33) | [Test, Order(2)]
method ThenOneExecutionFailed (line 39) | [Test, Order(3)]
method ThenFourExecutionsBlocked (line 45) | [Test, Order(4)]
method ThenOneExecutionLogRecord (line 51) | [Test, Order(5)]
method ThenTwoErrorLogRecords (line 57) | [Test, Order(6)]
method TearDown (line 64) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/02-Parent/GivenDependencyChainFailureHandlingAndRestart.cs
class GivenDependencyChainFailureHandlingAndRestart (line 8) | public class GivenDependencyChainFailureHandlingAndRestart
method WhenPipelineRun (line 13) | [OneTimeSetUp]
method ThenPipelineOutcomeIsSucceeded (line 32) | [Test]
method ThenOneExecutionLogFailedRecord (line 38) | [Test]
method ThenElevenExecutionLogSuccessRecord (line 43) | [Test]
method ThenTwoErrorLogRecords (line 49) | [Test]
method TearDown (line 56) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/02-Parent/GivenDisabledBatches.cs
class GivenDisabledBatches (line 8) | public class GivenDisabledBatches
method WhenPipelineRun (line 12) | [OneTimeSetUp]
method ThenPipelineOutcomeIsFailed (line 27) | [Test]
method TearDown (line 35) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/02-Parent/GivenDisabledPipelines.cs
class GivenDisabledPipelines (line 8) | class GivenDisabledPipelines
method WhenPipelineRun (line 12) | [OneTimeSetUp]
method ThenPipelineOutcomeIsFailed (line 26) | [Test, Order(1)]
method TearDown (line 34) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/02-Parent/GivenDisabledStages.cs
class GivenDisabledStages (line 8) | public class GivenDisabledStages
method WhenPipelineRun (line 12) | [OneTimeSetUp]
method ThenPipelineOutcomeIsFailed (line 26) | [Test, Order(1)]
method TearDown (line 34) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/02-Parent/GivenNoErrorsAndSPNStoredInDatabase.cs
class GivenNoErrorsAndSPNStoredInDatabase (line 8) | public class GivenNoErrorsAndSPNStoredInDatabase
method WhenPipelineRun (line 12) | [OneTimeSetUp]
method ThenPipelineOutcomeIsSucceeded (line 27) | [Test]
method ThenCurrentExecutionTableIsEmpty (line 37) | [Test]
method ThenElevenExecutionLogRecords (line 43) | [Test]
method TearDown (line 51) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/02-Parent/GivenNoErrorsAndSPNStoredInKeyVault.cs
class GivenNoErrorsAndSPNStoredInKeyVault (line 8) | public class GivenNoErrorsAndSPNStoredInKeyVault
method WhenPipelineRun (line 12) | [OneTimeSetUp]
method ThenPipelineOutcomeIsSucceeded (line 27) | [Test, Order(1)]
method ThenCurrentExecutionTableIsEmpty (line 37) | [Test, Order(2)]
method ThenElevenExecutionLogRecords (line 43) | [Test, Order(3)]
method TearDown (line 51) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/02-Parent/GivenNoFailureHandling.cs
class GivenNoFailureHandling (line 8) | public class GivenNoFailureHandling
method WhenPipelineRun (line 12) | [OneTimeSetUp]
method ThenPipelineOutcomeIsFailed (line 27) | [Test, Order(1)]
method ThenTenExecutionsSucceeded (line 33) | [Test, Order(2)]
method ThenOneExecutionFailed (line 39) | [Test, Order(3)]
method ThenOneExecutionLogRecord (line 45) | [Test, Order(4)]
method ThenTwoErrorLogRecords (line 51) | [Test, Order(5)]
method TearDown (line 59) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/02-Parent/GivenNoPipelineParameters.cs
class GivenNoPipelineParameters (line 8) | class GivenNoPipelineParameters
method WhenPipelineRun (line 12) | [OneTimeSetUp]
method ThenPipelineOutcomeIsSucceeded (line 27) | [Test]
method TearDown (line 39) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/02-Parent/GivenOneExecutionStage.cs
class GivenOneExecutionStage (line 8) | class GivenOneExecutionStage
method WhenPipelineRun (line 12) | [OneTimeSetUp]
method ThenPipelineOutcomeIsSucceeded (line 28) | [Test]
method TearDown (line 36) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/02-Parent/GivenSimpleFailureHandling.cs
class GivenSimpleFailureHandling (line 8) | public class GivenSimpleFailureHandling
method WhenPipelineRun (line 12) | [OneTimeSetUp]
method ThenPipelineOutcomeIsFailed (line 27) | [Test, Order(1)]
method ThenThreeExecutionsSucceeded (line 33) | [Test, Order(2)]
method ThenOneExecutionFailed (line 39) | [Test, Order(3)]
method ThenSevenExecutionsBlocked (line 45) | [Test, Order(4)]
method ThenOneExecutionLogRecord (line 51) | [Test, Order(5)]
method ThenTwoErrorLogRecords (line 57) | [Test, Order(6)]
method TearDown (line 64) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/02-Parent/GivenSimpleFailureHandlingAndRestart.cs
class GivenSimpleFailureHandlingAndRestart (line 8) | public class GivenSimpleFailureHandlingAndRestart
method WhenPipelineRun (line 13) | [OneTimeSetUp]
method ThenPipelineOutcomeIsSucceeded (line 32) | [Test]
method ThenOneExecutionLogFailedRecord (line 38) | [Test]
method ThenElevenExecutionLogSuccessRecord (line 43) | [Test]
method ThenTwoErrorLogRecords (line 49) | [Test]
method TearDown (line 56) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/02-Parent/GivenSimpleFailureHandlingAndRestartOveride.cs
class GivenSimpleFailureHandlingAndRestartOveride (line 8) | public class GivenSimpleFailureHandlingAndRestartOveride
method WhenPipelineRun (line 13) | [OneTimeSetUp]
method ThenPipelineOutcomeIsSucceeded (line 32) | [Test]
method ThenOneExecutionLogFailedRecord (line 38) | [Test]
method ThenFourthteenExecutionLogSuccessRecord (line 43) | [Test]
method ThenSevenExecutionsBlocked (line 48) | [Test]
method ThenTwoErrorLogRecords (line 53) | [Test]
method TearDown (line 60) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/02-Parent/ParentHelper.cs
class ParentHelper (line 10) | class ParentHelper : CoverageHelper<ParentHelper>
method RunPipeline (line 12) | public async Task RunPipeline()
method RunPipeline (line 16) | public async Task RunPipeline(int fakeDelayMilliseconds)
method CancelAnyWorkerPipeline (line 22) | public async Task CancelAnyWorkerPipeline()
method CancelIntentionalErrorWorkerPipeline (line 26) | public async Task CancelIntentionalErrorWorkerPipeline()
method RunAsync (line 31) | public virtual Task RunAsync()
method WithTenantAndSubscriptionIds (line 35) | public ParentHelper WithTenantAndSubscriptionIds()
method WithSPNInDatabase (line 40) | public ParentHelper WithSPNInDatabase(string workerFactoryName)
method WithSPNInKeyVault (line 46) | public ParentHelper WithSPNInKeyVault(string workerFactoryName)
method WithBasicMetadata (line 52) | public ParentHelper WithBasicMetadata()
method WithEmptyExecutionTables (line 58) | public ParentHelper WithEmptyExecutionTables()
method WithRunningPipelineStatusInPlaceOf (line 67) | public ParentHelper WithRunningPipelineStatusInPlaceOf(string statusTo...
method WithSimulatedError (line 73) | public ParentHelper WithSimulatedError()
method WithoutSimulatedError (line 79) | public ParentHelper WithoutSimulatedError()
method WithBatchesDisabled (line 85) | public ParentHelper WithBatchesDisabled()
method WithStagesDisabled (line 91) | public ParentHelper WithStagesDisabled()
method WithStagesEnabled (line 97) | public ParentHelper WithStagesEnabled()
method WithOnlyStageOneEnabled (line 103) | public ParentHelper WithOnlyStageOneEnabled()
method WithPipelinesDisabled (line 112) | public ParentHelper WithPipelinesDisabled()
method WithPipelinesEnabled (line 117) | public ParentHelper WithPipelinesEnabled()
method With2MinWaitsOnWorkers (line 122) | public ParentHelper With2MinWaitsOnWorkers()
method WithBatchExecutionHandling (line 128) | public ParentHelper WithBatchExecutionHandling()
method WithoutBatchExecutionHandling (line 136) | public ParentHelper WithoutBatchExecutionHandling()
method WithFailureHandling (line 144) | public ParentHelper WithFailureHandling(string mode)
method WithCancelledWorkersBlock (line 152) | public ParentHelper WithCancelledWorkersBlock(bool mode)
method WithOverideRestart (line 160) | public ParentHelper WithOverideRestart(bool mode)
method WithoutPrecursorObject (line 169) | public ParentHelper WithoutPrecursorObject()
method WithPrecursorObject (line 177) | public ParentHelper WithPrecursorObject()
method WithSingleExecutionStage (line 184) | public ParentHelper WithSingleExecutionStage()
method WithCustom (line 190) | public ParentHelper WithCustom()
method SetFalsePipelineStatus (line 196) | private ParentHelper SetFalsePipelineStatus(string falseStatus, string...
method GetWorkerRunId (line 202) | private string GetWorkerRunId(string pipelineName = null)
method EnableDisableMetadata (line 220) | private void EnableDisableMetadata(string table, bool state)
method EnableDisableMetadata (line 226) | private void EnableDisableMetadata(string table, bool state, string wh...
method SetParameterValue (line 231) | private void SetParameterValue(string value, string where, string equals)
method SimulateError (line 237) | private void SimulateError(bool simulate)
method TearDown (line 247) | public override void TearDown()
FILE: FactoryTesting/Pipelines/Utilities/GivenCheckForRunningPipeline.cs
class GivenCheckForRunningPipeline (line 9) | public class GivenCheckForRunningPipeline
method WhenPipelineRun (line 13) | [OneTimeSetUp]
method ThenPipelineOutcomeIsFailed (line 24) | [Test]
method ThenActivityShouldReturnOneFilteredItemCount (line 30) | [Test]
method ThenActivityShouldReturnMatchingSubscriptionId (line 37) | [Test]
method ThenActivityShouldReturnMatchingResourceGroup (line 46) | [Test]
method TearDown (line 55) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/Utilities/GivenEmailSender.cs
class GivenEmailSender (line 8) | public class GivenEmailSender
method WhenPipelineRun (line 12) | [OneTimeSetUp]
method ThenPipelineOutcomeIsSucceeded (line 23) | [Test]
method ThenActivityShouldReturnEmailSentTrue (line 29) | [Test]
method TearDown (line 36) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/Utilities/GivenThrowException.cs
class GivenThrowException (line 8) | public class GivenThrowException
method WhenPipelineRun (line 12) | [OneTimeSetUp]
method ThenPipelineOutcomeIsFailed (line 21) | [Test]
method TearDown (line 27) | [OneTimeTearDown]
FILE: FactoryTesting/Pipelines/Utilities/UtilitiesHelper.cs
class UtilitiesHelper (line 10) | class UtilitiesHelper : CoverageHelper<UtilitiesHelper>
method WithBasicMetadata (line 12) | public UtilitiesHelper WithBasicMetadata()
method WithTenantAndSubscriptionIds (line 18) | public UtilitiesHelper WithTenantAndSubscriptionIds()
method TearDown (line 24) | public override void TearDown()
FILE: Functions/Functions/CancelPipeline.cs
class CancelPipeline (line 15) | public static class CancelPipeline
method Run (line 17) | [FunctionName("CancelPipeline")]
FILE: Functions/Functions/CheckPipelineStatus.cs
class CheckPipelineStatus (line 15) | public static class CheckPipelineStatus
method Run (line 17) | [FunctionName("CheckPipelineStatus")]
FILE: Functions/Functions/ExecutePipeline.cs
class ExecutePipeline (line 15) | public static class ExecutePipeline
method Run (line 17) | [FunctionName("ExecutePipeline")]
FILE: Functions/Functions/GetActivityErrors.cs
class GetActivityErrors (line 15) | public static class GetActivityErrors
method Run (line 17) | [FunctionName("GetActivityErrors")]
FILE: Functions/Functions/SendEmail.cs
class SendEmail (line 17) | public static class SendEmail
method Run (line 19) | [FunctionName("SendEmail")]
FILE: Functions/Functions/ValidatePipeline.cs
class ValidatePipeline (line 15) | public static class ValidatePipeline
method Run (line 17) | [FunctionName("ValidatePipeline")]
FILE: Functions/Helpers/BodyReader.cs
class BodyReader (line 12) | public class BodyReader
method BodyReader (line 15) | public BodyReader(HttpRequest httpRequest)
method GetRequestBody (line 20) | public Task<PipelineRequest> GetRequestBody()
method GetRequestBodyAsync (line 26) | public async Task<PipelineRequest> GetRequestBodyAsync()
method GetRunRequestBody (line 32) | public Task<PipelineRunRequest> GetRunRequestBody()
method GetRunRequestBodyAsync (line 38) | public async Task<PipelineRunRequest> GetRunRequestBodyAsync()
FILE: Functions/Helpers/InvalidRequestException.cs
class InvalidRequestException (line 6) | [Serializable]
method InvalidRequestException (line 9) | public InvalidRequestException()
method InvalidRequestException (line 13) | public InvalidRequestException(string message) : base(message)
method InvalidRequestException (line 17) | public InvalidRequestException(string message, Exception innerExceptio...
method InvalidRequestException (line 21) | protected InvalidRequestException(SerializationInfo info, StreamingCon...
FILE: Functions/Helpers/KeyVaultClient.cs
class KeyVaultClient (line 7) | internal class KeyVaultClient
method GetSecretFromUri (line 11) | public static string GetSecretFromUri(string secretString)
method GetSecretFromUri (line 16) | public static string GetSecretFromUri(Uri secretUri)
method GetSecretFromName (line 23) | public static string GetSecretFromName(string keyVaultURL, string secr...
method CreateKeyVaultClient (line 28) | private static SecretClient CreateKeyVaultClient(string keyVaultURL)
FILE: Functions/Helpers/PipelineRequest.cs
class PipelineRequest (line 7) | public class PipelineRequest
method Validate (line 20) | public virtual void Validate(ILogger logger)
method CheckUri (line 53) | private bool CheckUri(string uriValue)
method CheckGuid (line 61) | public bool CheckGuid(string idValue)
method ReportInvalidBody (line 68) | protected void ReportInvalidBody(ILogger logger)
method ReportInvalidBody (line 75) | protected void ReportInvalidBody(ILogger logger, string additions)
FILE: Functions/Helpers/PipelineRunRequest.cs
class PipelineRunRequest (line 6) | public class PipelineRunRequest : PipelineRequest
method Validate (line 15) | public override void Validate(ILogger logger)
FILE: Functions/Helpers/SMTPClient.cs
class SMTPClient (line 6) | internal class SMTPClient
method CreateSMTPClient (line 10) | public static SmtpClient CreateSMTPClient()
FILE: Functions/Services/AzureDataFactoryService.cs
class AzureDataFactoryService (line 13) | public class AzureDataFactoryService : PipelineService
method AzureDataFactoryService (line 18) | public AzureDataFactoryService(PipelineRequest request, ILogger logger)
method ValidatePipeline (line 36) | public override PipelineDescription ValidatePipeline(PipelineRequest r...
method ExecutePipeline (line 79) | public override PipelineRunStatus ExecutePipeline(PipelineRequest requ...
method CancelPipeline (line 123) | public override PipelineRunStatus CancelPipeline(PipelineRunRequest re...
method GetPipelineRunStatus (line 182) | public override PipelineRunStatus GetPipelineRunStatus(PipelineRunRequ...
method GetPipelineRunActivityErrors (line 212) | public override PipelineErrorDetail GetPipelineRunActivityErrors(Pipel...
method Dispose (line 284) | public override void Dispose()
FILE: Functions/Services/AzureSynapseService.cs
class AzureSynapseService (line 17) | public class AzureSynapseService : PipelineService
method AzureSynapseService (line 24) | public AzureSynapseService(PipelineRequest request, ILogger logger)
method ValidatePipeline (line 54) | public override PipelineDescription ValidatePipeline(PipelineRequest r...
method ExecutePipeline (line 108) | public override PipelineRunStatus ExecutePipeline(PipelineRequest requ...
method CancelPipeline (line 149) | public override PipelineRunStatus CancelPipeline(PipelineRunRequest re...
method GetPipelineRunStatus (line 202) | public override PipelineRunStatus GetPipelineRunStatus(PipelineRunRequ...
method GetPipelineRunActivityErrors (line 227) | public override PipelineErrorDetail GetPipelineRunActivityErrors(Pipel...
method Dispose (line 294) | public override void Dispose()
FILE: Functions/Services/PipelineService.cs
class PipelineService (line 7) | public abstract class PipelineService : IDisposable
method GetServiceForRequest (line 11) | public static PipelineService GetServiceForRequest(PipelineRequest pr,...
method ValidatePipeline (line 22) | public abstract PipelineDescription ValidatePipeline(PipelineRequest r...
method ExecutePipeline (line 24) | public abstract PipelineRunStatus ExecutePipeline(PipelineRequest requ...
method CancelPipeline (line 26) | public abstract PipelineRunStatus CancelPipeline(PipelineRunRequest re...
method GetPipelineRunStatus (line 28) | public abstract PipelineRunStatus GetPipelineRunStatus(PipelineRunRequ...
method GetPipelineRunActivityErrors (line 30) | public abstract PipelineErrorDetail GetPipelineRunActivityErrors(Pipel...
method PipelineNameCheck (line 32) | protected void PipelineNameCheck(string requestName, string foundName)
method Dispose (line 40) | public abstract void Dispose();
FILE: Functions/Services/PipelineServiceType.cs
type PipelineServiceType (line 3) | public enum PipelineServiceType
FILE: Functions/Services/Returns/PipelineDescription.cs
class PipelineDescription (line 3) | public class PipelineDescription
FILE: Functions/Services/Returns/PipelineErrorDetail.cs
class PipelineErrorDetail (line 5) | public class PipelineErrorDetail : PipelineRunStatus
method PipelineErrorDetail (line 7) | public PipelineErrorDetail()
class FailedActivity (line 22) | public class FailedActivity
FILE: Functions/Services/Returns/PipelineRunStatus.cs
class PipelineRunStatus (line 5) | public class PipelineRunStatus
method ConvertPipelineStatus (line 22) | private string ConvertPipelineStatus(string actualStatus)
FILE: MetadataDB/dbo/Tables/ServicePrincipals.sql
type dbo (line 1) | CREATE TABLE [dbo].[ServicePrincipals](
FILE: MetadataDB/procfwk/Functions/GetPropertyValueInternal.sql
function procfwk (line 1) | CREATE FUNCTION [procfwk].[GetPropertyValueInternal]
FILE: MetadataDB/procfwk/Tables/BatchExecution.sql
type procfwk (line 1) | CREATE TABLE [procfwk].[BatchExecution](
FILE: MetadataDB/procfwk/Tables/BatchStageLink.sql
type procfwk (line 1) | CREATE TABLE [procfwk].[BatchStageLink]
FILE: MetadataDB/procfwk/Tables/Batches.sql
type procfwk (line 1) | CREATE TABLE [procfwk].[Batches](
FILE: MetadataDB/procfwk/Tables/CurrentExecution.sql
type procfwk (line 1) | CREATE TABLE [procfwk].[CurrentExecution] (
type IDX_GetPipelinesInStage (line 21) | CREATE NONCLUSTERED INDEX [IDX_GetPipelinesInStage] ON [procfwk].[Curren...
FILE: MetadataDB/procfwk/Tables/ErrorLog.sql
type procfwk (line 1) | CREATE TABLE [procfwk].[ErrorLog]
FILE: MetadataDB/procfwk/Tables/ExecutionLog.sql
type procfwk (line 1) | CREATE TABLE [procfwk].[ExecutionLog]
FILE: MetadataDB/procfwk/Tables/Orchestrators.sql
type procfwk (line 1) | CREATE TABLE [procfwk].[Orchestrators]
FILE: MetadataDB/procfwk/Tables/PipelineAlertLink.sql
type procfwk (line 1) | CREATE TABLE [procfwk].[PipelineAlertLink]
FILE: MetadataDB/procfwk/Tables/PipelineAuthLink.sql
type procfwk (line 1) | CREATE TABLE [procfwk].[PipelineAuthLink]
FILE: MetadataDB/procfwk/Tables/PipelineDependencies.sql
type procfwk (line 1) | CREATE TABLE [procfwk].[PipelineDependencies]
FILE: MetadataDB/procfwk/Tables/PipelineParameters.sql
type procfwk (line 1) | CREATE TABLE [procfwk].[PipelineParameters] (
FILE: MetadataDB/procfwk/Tables/Pipelines.sql
type procfwk (line 1) | CREATE TABLE [procfwk].[Pipelines] (
FILE: MetadataDB/procfwk/Tables/Properties.sql
type procfwk (line 1) | CREATE TABLE [procfwk].[Properties]
FILE: MetadataDB/procfwk/Tables/Recipients.sql
type procfwk (line 1) | CREATE TABLE [procfwk].[Recipients]
FILE: MetadataDB/procfwk/Tables/Stages.sql
type procfwk (line 1) | CREATE TABLE [procfwk].[Stages] (
FILE: MetadataDB/procfwk/Tables/Subscriptions.sql
type procfwk (line 1) | CREATE TABLE [procfwk].[Subscriptions]
FILE: MetadataDB/procfwk/Views/CurrentProperties.sql
type procfwk (line 1) | CREATE VIEW [procfwk].[CurrentProperties]
FILE: MetadataDB/procfwk/Views/DataFactorys.sql
type procfwk (line 1) | CREATE VIEW [procfwk].[DataFactorys]
FILE: MetadataDB/procfwk/Views/PipelineParameterDataSizes.sql
type procfwk (line 1) | CREATE VIEW [procfwk].[PipelineParameterDataSizes]
FILE: MetadataDB/procfwkHelpers/Views/PipelineDependencyChains.sql
type procfwkHelpers (line 1) | CREATE VIEW [procfwkHelpers].[PipelineDependencyChains]
FILE: MetadataDB/procfwkReporting/Views/AverageStageDuration.sql
type procfwkReporting (line 1) | CREATE VIEW [procfwkReporting].[AverageStageDuration]
FILE: MetadataDB/procfwkReporting/Views/CompleteExecutionErrorLog.sql
type procfwkReporting (line 1) | CREATE VIEW [procfwkReporting].[CompleteExecutionErrorLog]
FILE: MetadataDB/procfwkReporting/Views/CompleteExecutionLog.sql
type procfwkReporting (line 1) | CREATE VIEW [procfwkReporting].[CompleteExecutionLog]
FILE: MetadataDB/procfwkReporting/Views/CurrentExecutionSummary.sql
type procfwkReporting (line 1) | CREATE VIEW [procfwkReporting].[CurrentExecutionSummary]
FILE: MetadataDB/procfwkReporting/Views/LastExecution.sql
type procfwkReporting (line 1) | CREATE VIEW [procfwkReporting].[LastExecution]
FILE: MetadataDB/procfwkReporting/Views/LastExecutionSummary.sql
type procfwkReporting (line 1) | CREATE VIEW [procfwkReporting].[LastExecutionSummary]
FILE: MetadataDB/procfwkReporting/Views/WorkerParallelismOverTime.sql
type procfwkReporting (line 1) | CREATE VIEW [procfwkReporting].[WorkerParallelismOverTime]
Condensed preview — 344 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (3,014K chars).
[
{
"path": ".gitattributes",
"chars": 66,
"preview": "# Auto detect text files and perform LF normalization\n* text=auto\n"
},
{
"path": ".github/FUNDING.yml",
"chars": 70,
"preview": "# These are supported funding model platforms\n\ngithub: [mrpaulandrew]\n"
},
{
"path": ".github/ISSUE_TEMPLATE/bug-found.md",
"chars": 728,
"preview": "---\nname: Bug Found\nabout: Tell me about the bug you've found in the procfwk please\ntitle: ''\nlabels: bug\nassignees: mrp"
},
{
"path": ".github/ISSUE_TEMPLATE/feature-request.md",
"chars": 192,
"preview": "---\nname: Feature Request\nabout: Tell me about your idea to enhance the procfwk please\ntitle: ''\nlabels: enhancement\nass"
},
{
"path": ".github/ISSUE_TEMPLATE/help---support-request.md",
"chars": 670,
"preview": "---\nname: Help & Support Request\nabout: Tell me about the problem or error you are facing when using the procfwk\ntitle: "
},
{
"path": ".gitignore",
"chars": 4409,
"preview": "################################################################################\n# This .gitignore file was automatical"
},
{
"path": ".vscode/extensions.json",
"chars": 97,
"preview": "{\n \"recommendations\": [\n \"ms-azuretools.vscode-azurefunctions\",\n \"ms-vscode.csharp\"\n ]\n}\n"
},
{
"path": ".vscode/launch.json",
"chars": 254,
"preview": "{\n \"version\": \"0.2.0\",\n \"configurations\": [\n {\n \"name\": \"Attach to .NET Functions\",\n "
},
{
"path": ".vscode/settings.json",
"chars": 275,
"preview": "{\n \"azureFunctions.deploySubpath\": \"PipelineExecutor/bin/Release/netcoreapp3.1/publish\",\n \"azureFunctions.projectL"
},
{
"path": ".vscode/tasks.json",
"chars": 2391,
"preview": "{\n \"version\": \"2.0.0\",\n \"tasks\": [\n {\n \"label\": \"clean\",\n \"command\": \"dotnet\",\n "
},
{
"path": "ARM Templates/Data Factory/v1.0 Export.json",
"chars": 37107,
"preview": "{\n \"$schema\": \"http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#\",\n \"contentVersion\":"
},
{
"path": "ARM Templates/Data Factory/v1.1 Export.json",
"chars": 43393,
"preview": "{\n \"$schema\": \"http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#\",\n \"contentVersion\":"
},
{
"path": "ARM Templates/Data Factory/v1.2 Export.json",
"chars": 69214,
"preview": "{\n \"$schema\": \"http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#\",\n \"contentVersion\":"
},
{
"path": "ARM Templates/Data Factory/v1.3 Export.json",
"chars": 74757,
"preview": "{\n \"$schema\": \"http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#\",\n \"contentVersion\":"
},
{
"path": "ARM Templates/Data Factory/v1.4 Export.json",
"chars": 90243,
"preview": "{\n \"$schema\": \"http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#\",\n \"contentVersion\":"
},
{
"path": "ARM Templates/Data Factory/v1.5 Export.json",
"chars": 98229,
"preview": "{\n \"$schema\": \"http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#\",\n \"contentVersion\":"
},
{
"path": "ARM Templates/Data Factory/v1.6 Export.json",
"chars": 103407,
"preview": "{\n \"$schema\": \"http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#\",\n \"contentVersion\":"
},
{
"path": "ARM Templates/Data Factory/v1.7 Export.json",
"chars": 115845,
"preview": "{\n \"$schema\": \"http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#\",\n \"contentVersion\":"
},
{
"path": "ARM Templates/Data Factory/v1.8 Export.json",
"chars": 145248,
"preview": "{\n \"$schema\": \"http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#\",\n \"contentVersion\":"
},
{
"path": "ARM Templates/Data Factory/v1.8.3 Export.json",
"chars": 152438,
"preview": "{\n \"$schema\": \"http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#\",\n \"contentVersion\":"
},
{
"path": "ARM Templates/Data Factory/v1.8.5 Export.json",
"chars": 136911,
"preview": "{\n \"$schema\": \"http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#\",\n \"contentVersion\":"
},
{
"path": "ARM Templates/Data Factory/v1.8.6 Export.json",
"chars": 158083,
"preview": "{\n \"$schema\": \"http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#\",\n \"contentVersion\":"
},
{
"path": "ARM Templates/Data Factory/v1.9 Export.json",
"chars": 160066,
"preview": "{\n \"$schema\": \"http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#\",\n \"contentVersion\":"
},
{
"path": "ARM Templates/Data Factory/v1.9.1 Export.json",
"chars": 162857,
"preview": "{\n \"$schema\": \"http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#\",\n \"contentVersion\":"
},
{
"path": "ARM Templates/Data Factory/v1.9.2 Export.json",
"chars": 145880,
"preview": "{\n \"$schema\": \"http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#\",\n \"contentVersion\": \"1."
},
{
"path": "ARM Templates/Data Factory/v2.0 Export.json",
"chars": 214866,
"preview": "{\n \"$schema\": \"http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#\",\n \"contentVersion\":"
},
{
"path": "ARM Templates/Functions App/v1.6 Export.json",
"chars": 9861,
"preview": "{\n \"$schema\": \"https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#\",\n \"contentVersion\": \"1"
},
{
"path": "ARM Templates/SQL Database/v1.6 Export.json",
"chars": 7970,
"preview": "{\n \"$schema\": \"https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#\",\n \"contentVersion\": \"1"
},
{
"path": "ARM Templates/Synapse/v2.0 Export.json",
"chars": 150759,
"preview": "{\n \"$schema\": \"http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#\",\n \"contentVersion\": \"1."
},
{
"path": "CODE_OF_CONDUCT.md",
"chars": 3353,
"preview": "# Contributor Covenant Code of Conduct\n\n## Our Pledge\n\nIn the interest of fostering an open and welcoming environment, w"
},
{
"path": "CONTRIBUTING.md",
"chars": 179,
"preview": "If you'd like to contribute towards this code project please contact me. I kindly request a minimum of 5 years experienc"
},
{
"path": "DataFactory/dataset/GetSetMetadata.json",
"chars": 361,
"preview": "{\n\t\"name\": \"GetSetMetadata\",\n\t\"properties\": {\n\t\t\"description\": \"Single generic dataset used to get and set all database "
},
{
"path": "DataFactory/factory/FrameworkFactory.json",
"chars": 144,
"preview": "{\n\t\"name\": \"FrameworkFactory\",\n\t\"properties\": {\n\t\t\"globalConfigurations\": {\n\t\t\t\"PipelineBillingEnabled\": \"true\"\n\t\t}\n\t},\n"
},
{
"path": "DataFactory/integrationRuntime/AzureIR-UKSouth.json",
"chars": 288,
"preview": "{\n\t\"name\": \"AzureIR-UKSouth\",\n\t\"properties\": {\n\t\t\"type\": \"Managed\",\n\t\t\"typeProperties\": {\n\t\t\t\"computeProperties\": {\n\t\t\t\t"
},
{
"path": "DataFactory/linkedService/FrameworkFunctions.json",
"chars": 579,
"preview": "{\n\t\"name\": \"FrameworkFunctions\",\n\t\"properties\": {\n\t\t\"annotations\": [\n\t\t\t\"procfwk\"\n\t\t],\n\t\t\"type\": \"AzureFunction\",\n\t\t\"typ"
},
{
"path": "DataFactory/linkedService/Keys.json",
"chars": 316,
"preview": "{\n\t\"name\": \"Keys\",\n\t\"properties\": {\n\t\t\"annotations\": [\n\t\t\t\"procfwk\"\n\t\t],\n\t\t\"type\": \"AzureKeyVault\",\n\t\t\"typeProperties\": "
},
{
"path": "DataFactory/linkedService/SupportDatabase.json",
"chars": 424,
"preview": "{\n\t\"name\": \"SupportDatabase\",\n\t\"properties\": {\n\t\t\"description\": \"Connection between ADF and processing framework metadat"
},
{
"path": "DataFactory/pipeline/01-Grandparent.json",
"chars": 868,
"preview": "{\n\t\"name\": \"01-Grandparent\",\n\t\"properties\": {\n\t\t\"description\": \"procfwk grandparent pipeline used optionally to bootstra"
},
{
"path": "DataFactory/pipeline/02-Parent.json",
"chars": 28583,
"preview": "{\n\t\"name\": \"02-Parent\",\n\t\"properties\": {\n\t\t\"description\": \"ADF.procfwk parent pipeline used to bootstrap the orchestrati"
},
{
"path": "DataFactory/pipeline/03-Child.json",
"chars": 3006,
"preview": "{\n\t\"name\": \"03-Child\",\n\t\"properties\": {\n\t\t\"description\": \"procfwk child pipeline used to execute Worker pipelines within"
},
{
"path": "DataFactory/pipeline/04-Infant.json",
"chars": 38975,
"preview": "{\n\t\"name\": \"04-Infant\",\n\t\"properties\": {\n\t\t\"description\": \"procfwk infant pipeline used to check when the processing pip"
},
{
"path": "DataFactory/pipeline/Check For Running Pipeline.json",
"chars": 18549,
"preview": "{\n\t\"name\": \"Check For Running Pipeline\",\n\t\"properties\": {\n\t\t\"description\": \"For a given pipeline and optional batch name"
},
{
"path": "DataFactory/pipeline/Email Sender.json",
"chars": 1613,
"preview": "{\n\t\"name\": \"Email Sender\",\n\t\"properties\": {\n\t\t\"description\": \"Provide a simple abstract over the send email function wit"
},
{
"path": "DataFactory/pipeline/Intentional Error.json",
"chars": 1969,
"preview": "{\n\t\"name\": \"Intentional Error\",\n\t\"properties\": {\n\t\t\"description\": \"Used just so the procfwk has something to call during"
},
{
"path": "DataFactory/pipeline/Throw Exception.json",
"chars": 1759,
"preview": "{\n\t\"name\": \"Throw Exception\",\n\t\"properties\": {\n\t\t\"description\": \"Provide a simple way of throwing an exception within Da"
},
{
"path": "DataFactory/pipeline/Wait 1.json",
"chars": 624,
"preview": "{\n\t\"name\": \"Wait 1\",\n\t\"properties\": {\n\t\t\"description\": \"Used just so the procfwk has something to call during developmen"
},
{
"path": "DataFactory/pipeline/Wait 10.json",
"chars": 626,
"preview": "{\n\t\"name\": \"Wait 10\",\n\t\"properties\": {\n\t\t\"description\": \"Used just so the procfwk has something to call during developme"
},
{
"path": "DataFactory/pipeline/Wait 2.json",
"chars": 624,
"preview": "{\n\t\"name\": \"Wait 2\",\n\t\"properties\": {\n\t\t\"description\": \"Used just so the procfwk has something to call during developmen"
},
{
"path": "DataFactory/pipeline/Wait 3.json",
"chars": 624,
"preview": "{\n\t\"name\": \"Wait 3\",\n\t\"properties\": {\n\t\t\"description\": \"Used just so the procfwk has something to call during developmen"
},
{
"path": "DataFactory/pipeline/Wait 4.json",
"chars": 624,
"preview": "{\n\t\"name\": \"Wait 4\",\n\t\"properties\": {\n\t\t\"description\": \"Used just so the procfwk has something to call during developmen"
},
{
"path": "DataFactory/pipeline/Wait 5.json",
"chars": 624,
"preview": "{\n\t\"name\": \"Wait 5\",\n\t\"properties\": {\n\t\t\"description\": \"Used just so the procfwk has something to call during developmen"
},
{
"path": "DataFactory/pipeline/Wait 6.json",
"chars": 624,
"preview": "{\n\t\"name\": \"Wait 6\",\n\t\"properties\": {\n\t\t\"description\": \"Used just so the procfwk has something to call during developmen"
},
{
"path": "DataFactory/pipeline/Wait 7.json",
"chars": 624,
"preview": "{\n\t\"name\": \"Wait 7\",\n\t\"properties\": {\n\t\t\"description\": \"Used just so the procfwk has something to call during developmen"
},
{
"path": "DataFactory/pipeline/Wait 8.json",
"chars": 624,
"preview": "{\n\t\"name\": \"Wait 8\",\n\t\"properties\": {\n\t\t\"description\": \"Used just so the procfwk has something to call during developmen"
},
{
"path": "DataFactory/pipeline/Wait 9.json",
"chars": 625,
"preview": "{\n\t\"name\": \"Wait 9\",\n\t\"properties\": {\n\t\t\"description\": \"Used just so the procfwk has something to call during developmen"
},
{
"path": "DataFactory/trigger/FunctionalTestingTrigger.json",
"chars": 543,
"preview": "{\n\t\"name\": \"FunctionalTestingTrigger\",\n\t\"properties\": {\n\t\t\"description\": \"Used for functional testing of the framework i"
},
{
"path": "DeploymentTools/DataFactory/DeployProcFwkComponents.ps1",
"chars": 2586,
"preview": "function Publish-procfwkadf \n{\nParam(\n [Parameter(Mandatory)]\n [string]$resourceGroupName,\n [Parameter(Mandato"
},
{
"path": "DeploymentTools/DataFactory/DeployProcFwkComponents_Old.ps1",
"chars": 7403,
"preview": "# Set global variables as required:\n$resourceGroupName = \"ADF.procfwk\"\n$dataFactoryName = \"FrameworkFactoryDev\"\n$region "
},
{
"path": "DeploymentTools/DataFactory/Get Pipelines.ps1",
"chars": 913,
"preview": "# Set global variables as required:\n$resourceGroupName = \"ADF.procfwk\"\n$dataFactoryName = \"FrameworkFactory\"\n$region = "
},
{
"path": "DeploymentTools/DataFactory/GlobalVars.ps1",
"chars": 634,
"preview": "# Set global variables as required:\n$resourceGroupName = \"ADF.procfwk\"\n$dataFactoryName = \"FrameworkFactory\"\n$region = \""
},
{
"path": "DeploymentTools/DataFactory/PopulatePipelinesInDb.ps1",
"chars": 992,
"preview": "Param(\n [Parameter(Mandatory)]\n [string]$SqlServerName,\n [Parameter(Mandatory)]\n [string]$SqlDatabaseName,\n "
},
{
"path": "DeploymentTools/DataFactory/ProcFwkComponents.json",
"chars": 548,
"preview": "{\n \"linkedServices\": [\n \"/linkedService/Keys.json\",\n \"/linkedService/FrameworkFunctions.json\",\n \"/linkedServi"
},
{
"path": "DeploymentTools/DataFactory/config-all.csv",
"chars": 113,
"preview": "type,name,path,value\nlinkedService,SupportDatabase,typeProperties.connectionString.secretName,$($Env:SQLDatabase)"
},
{
"path": "DeploymentTools/Deployment.targets",
"chars": 5694,
"preview": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<Project ToolsVersion=\"4.0\" xmlns=\"http://schemas.microsoft.com/developer/msbuild"
},
{
"path": "DeploymentTools/DeploymentTools.deployproj",
"chars": 1719,
"preview": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<Project ToolsVersion=\"4.0\" xmlns=\"http://schemas.microsoft.com/developer/msbuil"
},
{
"path": "FactoryTesting/FactoryTesting.csproj",
"chars": 1323,
"preview": "<Project Sdk=\"Microsoft.NET.Sdk\">\n\n <PropertyGroup>\n <TargetFramework>netcoreapp3.1</TargetFramework>\n\n <IsPackab"
},
{
"path": "FactoryTesting/Helpers/CoverageHelper.cs",
"chars": 3323,
"preview": "using FactoryTesting.Helpers;\nusing Microsoft.Azure.Management.DataFactory.Models;\nusing NUnit.Framework;\nusing System;"
},
{
"path": "FactoryTesting/Helpers/DataFactoryHelper.cs",
"chars": 6438,
"preview": "using Microsoft.Azure.Management.DataFactory;\nusing Microsoft.Azure.Management.DataFactory.Models;\nusing Microsoft.Iden"
},
{
"path": "FactoryTesting/Helpers/DatabaseHelper.cs",
"chars": 5198,
"preview": "using System.Collections.Generic;\nusing System.Data;\nusing System.Data.SqlClient;\n\nnamespace FactoryTesting.Helpers\n{\n "
},
{
"path": "FactoryTesting/Helpers/PipelineRunHelper.cs",
"chars": 2932,
"preview": "using Microsoft.Azure.Management.DataFactory.Models;\nusing Newtonsoft.Json.Linq;\nusing System;\nusing System.Collections"
},
{
"path": "FactoryTesting/Helpers/SettingsHelper.cs",
"chars": 1363,
"preview": "using Azure.Identity;\nusing Azure.Security.KeyVault.Secrets;\nusing NUnit.Framework;\nusing System;\n\nnamespace FactoryTes"
},
{
"path": "FactoryTesting/Pipelines/01-Grandparent/Given300WorkerPipelines.cs",
"chars": 1253,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace"
},
{
"path": "FactoryTesting/Pipelines/01-Grandparent/Given600WorkerPipelineConcurrentBatches.cs",
"chars": 2192,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace"
},
{
"path": "FactoryTesting/Pipelines/01-Grandparent/GivenOneWorkerPipeline.cs",
"chars": 1030,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace"
},
{
"path": "FactoryTesting/Pipelines/01-Grandparent/GrandparentHelper.cs",
"chars": 2705,
"preview": "using FactoryTesting.Helpers;\nusing System.Collections.Generic;\nusing System.Threading.Tasks;\n\nnamespace FactoryTesting"
},
{
"path": "FactoryTesting/Pipelines/02-Parent/Given20ConcurrentBatchesFor1000WorkerPipelines.cs",
"chars": 9517,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace"
},
{
"path": "FactoryTesting/Pipelines/02-Parent/GivenAlreadyRunning.cs",
"chars": 1709,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace"
},
{
"path": "FactoryTesting/Pipelines/02-Parent/GivenBatchExecutionsForConcurrentBatches.cs",
"chars": 2235,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace"
},
{
"path": "FactoryTesting/Pipelines/02-Parent/GivenBatchExecutionsForConcurrentBatchesAlreadyRunning.cs",
"chars": 2277,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace"
},
{
"path": "FactoryTesting/Pipelines/02-Parent/GivenBatchExecutionsForConcurrentBatchesWithSimpleFailureHandling.cs",
"chars": 3436,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace"
},
{
"path": "FactoryTesting/Pipelines/02-Parent/GivenBatchExecutionsForConcurrentBatchesWithSimpleFailureHandlingAndRestart.cs",
"chars": 3895,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace"
},
{
"path": "FactoryTesting/Pipelines/02-Parent/GivenBatchExecutionsForConcurrentBatchesWithSimpleFailureHandlingAndRestartOveride.cs",
"chars": 4352,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace"
},
{
"path": "FactoryTesting/Pipelines/02-Parent/GivenBatchExecutionsForSingleBatch.cs",
"chars": 1676,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace"
},
{
"path": "FactoryTesting/Pipelines/02-Parent/GivenCancelledWorkerAndRestart.cs",
"chars": 1981,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace"
},
{
"path": "FactoryTesting/Pipelines/02-Parent/GivenCancelledWorkerInOneExecutionStage.cs",
"chars": 2024,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace"
},
{
"path": "FactoryTesting/Pipelines/02-Parent/GivenCancelledWorkerThatBlocks.cs",
"chars": 2189,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace"
},
{
"path": "FactoryTesting/Pipelines/02-Parent/GivenCancelledWorkerThatDoesntBlock.cs",
"chars": 1993,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace"
},
{
"path": "FactoryTesting/Pipelines/02-Parent/GivenCleanUpForCancelledWorkerAndRestart.cs",
"chars": 2367,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace"
},
{
"path": "FactoryTesting/Pipelines/02-Parent/GivenCleanUpForSuccessfulWorkersAndRestart.cs",
"chars": 2302,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace"
},
{
"path": "FactoryTesting/Pipelines/02-Parent/GivenDependencyChainFailureHandling.cs",
"chars": 2022,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace "
},
{
"path": "FactoryTesting/Pipelines/02-Parent/GivenDependencyChainFailureHandlingAndRestart.cs",
"chars": 1849,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace"
},
{
"path": "FactoryTesting/Pipelines/02-Parent/GivenDisabledBatches.cs",
"chars": 991,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace"
},
{
"path": "FactoryTesting/Pipelines/02-Parent/GivenDisabledPipelines.cs",
"chars": 952,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace"
},
{
"path": "FactoryTesting/Pipelines/02-Parent/GivenDisabledStages.cs",
"chars": 953,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace"
},
{
"path": "FactoryTesting/Pipelines/02-Parent/GivenNoErrorsAndSPNStoredInDatabase.cs",
"chars": 1390,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace "
},
{
"path": "FactoryTesting/Pipelines/02-Parent/GivenNoErrorsAndSPNStoredInKeyVault.cs",
"chars": 1421,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace"
},
{
"path": "FactoryTesting/Pipelines/02-Parent/GivenNoFailureHandling.cs",
"chars": 1797,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace "
},
{
"path": "FactoryTesting/Pipelines/02-Parent/GivenNoPipelineParameters.cs",
"chars": 1054,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace"
},
{
"path": "FactoryTesting/Pipelines/02-Parent/GivenOneExecutionStage.cs",
"chars": 1039,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace"
},
{
"path": "FactoryTesting/Pipelines/02-Parent/GivenSimpleFailureHandling.cs",
"chars": 2022,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace "
},
{
"path": "FactoryTesting/Pipelines/02-Parent/GivenSimpleFailureHandlingAndRestart.cs",
"chars": 1833,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace"
},
{
"path": "FactoryTesting/Pipelines/02-Parent/GivenSimpleFailureHandlingAndRestartOveride.cs",
"chars": 2091,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace"
},
{
"path": "FactoryTesting/Pipelines/02-Parent/ParentHelper.cs",
"chars": 8182,
"preview": "using FactoryTesting.Helpers;\nusing System;\nusing System.Data.SqlClient;\nusing System.Threading;\nusing System.Threading"
},
{
"path": "FactoryTesting/Pipelines/Utilities/GivenCheckForRunningPipeline.cs",
"chars": 1914,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\nusing Fact"
},
{
"path": "FactoryTesting/Pipelines/Utilities/GivenEmailSender.cs",
"chars": 1119,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace"
},
{
"path": "FactoryTesting/Pipelines/Utilities/GivenThrowException.cs",
"chars": 760,
"preview": "using FluentAssertions;\nusing NUnit.Framework;\nusing NUnit.Framework.Internal;\nusing System.Threading.Tasks;\n\nnamespace"
},
{
"path": "FactoryTesting/Pipelines/Utilities/UtilitiesHelper.cs",
"chars": 635,
"preview": "using FactoryTesting.Helpers;\nusing System;\nusing System.Data.SqlClient;\nusing System.Threading;\nusing System.Threading"
},
{
"path": "FactoryTesting/dev.runsettings",
"chars": 587,
"preview": "<?xml version='1.0' encoding='utf-8'?>\n\n<RunSettings>\n <TestRunParameters>\n <Parameter name=\"KeyVaultUrl\" valu"
},
{
"path": "FactoryTesting/multi.runsettings",
"chars": 593,
"preview": "<?xml version='1.0' encoding='utf-8'?>\n\n<RunSettings>\n <TestRunParameters>\n <Parameter name=\"KeyVaultUrl\" valu"
},
{
"path": "FactoryTesting/test.runsettings",
"chars": 587,
"preview": "<?xml version='1.0' encoding='utf-8'?>\n\n<RunSettings>\n <TestRunParameters>\n <Parameter name=\"KeyVaultUrl\" valu"
},
{
"path": "Functions/.vscode/extensions.json",
"chars": 102,
"preview": "{\n \"recommendations\": [\n \"ms-azuretools.vscode-azurefunctions\",\n \"ms-dotnettools.csharp\"\n ]\n}\n"
},
{
"path": "Functions/.vscode/launch.json",
"chars": 254,
"preview": "{\n \"version\": \"0.2.0\",\n \"configurations\": [\n {\n \"name\": \"Attach to .NET Functions\",\n "
},
{
"path": "Functions/.vscode/settings.json",
"chars": 258,
"preview": "{\n \"azureFunctions.deploySubpath\": \"bin/Release/netcoreapp3.1/publish\",\n \"azureFunctions.projectLanguage\": \"C#\",\n "
},
{
"path": "Functions/.vscode/tasks.json",
"chars": 1358,
"preview": "{\n\t\"version\": \"2.0.0\",\n\t\"tasks\": [\n\t\t{\n\t\t\t\"label\": \"clean\",\n\t\t\t\"command\": \"dotnet\",\n\t\t\t\"args\": [\n\t\t\t\t\"clean\",\n\t\t\t\t\"/prop"
},
{
"path": "Functions/Functions/CancelPipeline.cs",
"chars": 1328,
"preview": "using System;\nusing System.IO;\nusing System.Threading.Tasks;\nusing Microsoft.AspNetCore.Mvc;\nusing Microsoft.Azure.WebJo"
},
{
"path": "Functions/Functions/CheckPipelineStatus.cs",
"chars": 1361,
"preview": "using System;\nusing System.IO;\nusing System.Threading.Tasks;\nusing Microsoft.AspNetCore.Mvc;\nusing Microsoft.Azure.WebJo"
},
{
"path": "Functions/Functions/ExecutePipeline.cs",
"chars": 1327,
"preview": "using System;\nusing System.IO;\nusing System.Threading.Tasks;\nusing Microsoft.AspNetCore.Mvc;\nusing Microsoft.Azure.WebJo"
},
{
"path": "Functions/Functions/GetActivityErrors.cs",
"chars": 1363,
"preview": "using System;\nusing System.IO;\nusing System.Threading.Tasks;\nusing Microsoft.AspNetCore.Mvc;\nusing Microsoft.Azure.WebJo"
},
{
"path": "Functions/Functions/SendEmail.cs",
"chars": 6929,
"preview": "using System;\nusing System.IO;\nusing System.Threading.Tasks;\nusing System.Net.Mail;\nusing Microsoft.AspNetCore.Mvc;\nusin"
},
{
"path": "Functions/Functions/ValidatePipeline.cs",
"chars": 1341,
"preview": "using System;\nusing System.IO;\nusing System.Threading.Tasks;\nusing Microsoft.AspNetCore.Mvc;\nusing Microsoft.Azure.WebJo"
},
{
"path": "Functions/Functions.csproj",
"chars": 1290,
"preview": "<Project Sdk=\"Microsoft.NET.Sdk\">\n <PropertyGroup>\n <TargetFramework>netcoreapp3.1</TargetFramework>\n <AzureFunct"
},
{
"path": "Functions/Helpers/BodyReader.cs",
"chars": 1263,
"preview": "using System;\nusing System.Collections.Generic;\nusing System.IO;\nusing System.Text;\nusing System.Threading.Tasks;\nusing"
},
{
"path": "Functions/Helpers/InvalidRequestException.cs",
"chars": 608,
"preview": "using System;\nusing System.Runtime.Serialization;\n\nnamespace mrpaulandrew.azure.procfwk.Helpers\n{\n [Serializable]\n "
},
{
"path": "Functions/Helpers/KeyVaultClient.cs",
"chars": 1132,
"preview": "using Azure.Identity;\nusing Azure.Security.KeyVault.Secrets;\nusing System;\n\nnamespace mrpaulandrew.azure.procfwk.Helper"
},
{
"path": "Functions/Helpers/PipelineRequest.cs",
"chars": 3273,
"preview": "using Microsoft.Extensions.Logging;\nusing System;\nusing System.Collections.Generic;\n\nnamespace mrpaulandrew.azure.procf"
},
{
"path": "Functions/Helpers/PipelineRunRequest.cs",
"chars": 843,
"preview": "using System;\nusing Microsoft.Extensions.Logging;\n\nnamespace mrpaulandrew.azure.procfwk.Helpers\n{\n public class Pipe"
},
{
"path": "Functions/Helpers/SMTPClient.cs",
"chars": 1158,
"preview": "using System;\nusing System.Net.Mail;\n\nnamespace mrpaulandrew.azure.procfwk.Helpers\n{\n internal class SMTPClient\n "
},
{
"path": "Functions/Properties/ServiceDependencies/FrameworkSupportFunctions - Zip Deploy/profile.arm.json",
"chars": 6417,
"preview": "{\n \"$schema\": \"https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#\",\n \"content"
},
{
"path": "Functions/Properties/ServiceDependencies/FrameworkSupportFunctionsBig - Web Deploy/profile.arm.json",
"chars": 4269,
"preview": "{\n \"$schema\": \"https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#\",\n \"content"
},
{
"path": "Functions/Services/AzureDataFactoryService.cs",
"chars": 11153,
"preview": "using System;\nusing System.Threading;\nusing Newtonsoft.Json;\nusing Microsoft.Rest;\nusing Microsoft.Extensions.Logging;\n"
},
{
"path": "Functions/Services/AzureSynapseService.cs",
"chars": 11519,
"preview": "using System;\nusing System.Threading;\nusing System.Collections.Generic;\nusing Newtonsoft.Json;\nusing Microsoft.Rest;\nus"
},
{
"path": "Functions/Services/PipelineService.cs",
"chars": 1623,
"preview": "using mrpaulandrew.azure.procfwk.Helpers;\nusing Microsoft.Extensions.Logging;\nusing System;\n\nnamespace mrpaulandrew.azu"
},
{
"path": "Functions/Services/PipelineServiceType.cs",
"chars": 106,
"preview": "namespace mrpaulandrew.azure.procfwk\n{\n public enum PipelineServiceType\n {\n ADF, SYN\n }\n}"
},
{
"path": "Functions/Services/Returns/PipelineDescription.cs",
"chars": 342,
"preview": "namespace mrpaulandrew.azure.procfwk.Services\n{\n public class PipelineDescription\n {\n public string Pipeli"
},
{
"path": "Functions/Services/Returns/PipelineErrorDetail.cs",
"chars": 887,
"preview": "using System.Collections.Generic;\n\nnamespace mrpaulandrew.azure.procfwk.Services\n{\n public class PipelineErrorDetail"
},
{
"path": "Functions/Services/Returns/PipelineRunStatus.cs",
"chars": 946,
"preview": "using System.Collections.Generic;\n\nnamespace mrpaulandrew.azure.procfwk.Services\n{\n public class PipelineRunStatus\n "
},
{
"path": "Functions/host.json",
"chars": 24,
"preview": "{\n \"version\": \"2.0\"\n}"
},
{
"path": "Functions/template_local.settings.json",
"chars": 364,
"preview": "{\n \"IsEncrypted\": false,\n \"Values\": {\n \"AzureWebJobsStorage\": \"UseDevelopmentStorage=true\",\n \"FUNCTIONS_WORKER_R"
},
{
"path": "LICENSE",
"chars": 1378,
"preview": "MIT License\n\nCopyright (c) 2020 Paul Andrew (mrpaulandrew.com)\n\nPermission is hereby granted, free of charge, to any per"
},
{
"path": "MetadataDB/MetadataDB.refactorlog",
"chars": 629,
"preview": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<Operations Version=\"1.0\" xmlns=\"http://schemas.microsoft.com/sqlserver/dac/Seri"
},
{
"path": "MetadataDB/MetadataDB.sqlproj",
"chars": 15483,
"preview": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<Project DefaultTargets=\"Build\" xmlns=\"http://schemas.microsoft.com/developer/ms"
},
{
"path": "MetadataDB/Scripts/Alter Database Scale.sql",
"chars": 111,
"preview": "ALTER DATABASE [FrameworkMetadataDev] MODIFY (EDITION ='Standard', SERVICE_OBJECTIVE = 'S2', MAXSIZE = 50 GB);"
},
{
"path": "MetadataDB/Scripts/Handy Selects.sql",
"chars": 546,
"preview": "SELECT * FROM [procfwk].[BatchExecution]\n\nSELECT * FROM [procfwk].[CurrentExecution]\n\nSELECT * FROM [procfwkReporting]."
},
{
"path": "MetadataDB/Scripts/LogData/ErrorLogBackup.sql",
"chars": 251,
"preview": "IF OBJECT_ID(N'[dbo].[ErrorLogBackup]') IS NOT NULL DROP TABLE [dbo].[ErrorLogBackup];\n\nIF OBJECT_ID(N'[procfwk].[Error"
},
{
"path": "MetadataDB/Scripts/LogData/ErrorLogRestore.sql",
"chars": 2018,
"preview": "DECLARE @ErrColumns VARCHAR(MAX) = '';\nDECLARE @ErrValues VARCHAR(MAX) = '';\nDECLARE @ErrSQL VARCHAR(MAX) = '';\n\nIF EXI"
},
{
"path": "MetadataDB/Scripts/LogData/ExecutionLogBackup.sql",
"chars": 271,
"preview": "IF OBJECT_ID(N'[dbo].[ExecutionLogBackup]') IS NOT NULL DROP TABLE [dbo].[ExecutionLogBackup];\n\nIF OBJECT_ID(N'[procfwk"
},
{
"path": "MetadataDB/Scripts/LogData/ExecutionLogRestore.sql",
"chars": 1989,
"preview": "DECLARE @Columns VARCHAR(MAX) = '';\nDECLARE @Values VARCHAR(MAX) = '';\nDECLARE @SQL VARCHAR(MAX) = '';\n\nIF EXISTS\n\t(\n\tS"
},
{
"path": "MetadataDB/Scripts/Metadata/AlertOutcomes.sql",
"chars": 49,
"preview": "EXEC [procfwkHelpers].[SetDefaultAlertOutcomes];"
},
{
"path": "MetadataDB/Scripts/Metadata/DeleteAll.sql",
"chars": 3450,
"preview": "\t--BatchExecution\n\tIF OBJECT_ID(N'[procfwk].[BatchExecution]') IS NOT NULL \n\t\tBEGIN\n\t\t\tTRUNCATE TABLE [procfwk].[BatchE"
},
{
"path": "MetadataDB/Scripts/Metadata/DropLegacyObjects.sql",
"chars": 1867,
"preview": "IF OBJECT_ID(N'tempdb..#DropLegacyObjects') IS NOT NULL DROP PROCEDURE #DropLegacyObjects;\nGO\n\nCREATE PROCEDURE #DropLeg"
},
{
"path": "MetadataDB/Scripts/Metadata/DropLegacyTables.sql",
"chars": 1957,
"preview": "--PipelineProcesses\nIF EXISTS \n\t(\n\tSELECT\n\t\t* \n\tFROM\n\t\tsys.objects o\n\t\tINNER JOIN sys.schemas s\n\t\t\tON o.[schema_id] = s"
},
{
"path": "MetadataDB/Scripts/Metadata/Orchestrators.sql",
"chars": 139,
"preview": "EXEC [procfwkHelpers].[SetDefaultTenant];\nEXEC [procfwkHelpers].[SetDefaultSubscription];\nEXEC [procfwkHelpers].[SetDef"
},
{
"path": "MetadataDB/Scripts/Metadata/PipelineDependencies.sql",
"chars": 54,
"preview": "EXEC [procfwkHelpers].[SetDefaultPipelineDependants];"
},
{
"path": "MetadataDB/Scripts/Metadata/PipelineParams.sql",
"chars": 54,
"preview": "EXEC [procfwkHelpers].[SetDefaultPipelineParameters];"
},
{
"path": "MetadataDB/Scripts/Metadata/Pipelines.sql",
"chars": 45,
"preview": "EXEC [procfwkHelpers].[SetDefaultPipelines];"
},
{
"path": "MetadataDB/Scripts/Metadata/Properties.sql",
"chars": 46,
"preview": "EXEC [procfwkHelpers].[SetDefaultProperties];"
},
{
"path": "MetadataDB/Scripts/Metadata/RecipientAlertsLink.sql",
"chars": 58,
"preview": "EXEC [procfwkHelpers].[SetDefaultRecipientPipelineAlerts]"
},
{
"path": "MetadataDB/Scripts/Metadata/Recipients.sql",
"chars": 46,
"preview": "EXEC [procfwkHelpers].[SetDefaultRecipients];"
},
{
"path": "MetadataDB/Scripts/Metadata/ReplaceDataFactorys.sql",
"chars": 583,
"preview": "IF EXISTS \n\t(\n\tSELECT\n\t\t* \n\tFROM\n\t\tsys.objects o\n\t\tINNER JOIN sys.schemas s\n\t\t\tON o.[schema_id] = s.[schema_id]\n\tWHERE\n"
},
{
"path": "MetadataDB/Scripts/Metadata/Stages.sql",
"chars": 135,
"preview": "EXEC [procfwkHelpers].[SetDefaultBatches];\nEXEC [procfwkHelpers].[SetDefaultStages];\nEXEC [procfwkHelpers].[SetDefaultB"
},
{
"path": "MetadataDB/Scripts/Metadata/TransferHelperObjects.sql",
"chars": 1269,
"preview": "IF OBJECT_ID(N'tempdb..#TransferHelperObjects') IS NOT NULL DROP PROCEDURE #TransferHelperObjects;\nGO\n\nCREATE PROCEDURE "
},
{
"path": "MetadataDB/Scripts/Metadata/TransferReportingObjects.sql",
"chars": 643,
"preview": "IF OBJECT_ID(N'tempdb..#TransferReportingObjects') IS NOT NULL DROP PROCEDURE #TransferReportingObjects;\nGO\n\nCREATE PRO"
},
{
"path": "MetadataDB/Scripts/Script.PostDeployment.sql",
"chars": 1145,
"preview": "/*\nPost-Deployment Script Template\t\t\t\t\t\t\t\n-----------------------------------------------------------------------------"
},
{
"path": "MetadataDB/Scripts/Script.PreDeployment.sql",
"chars": 772,
"preview": "/*\n Pre-Deployment Script Template\t\t\t\t\t\t\t\n-----------------------------------------------------------------------------"
},
{
"path": "MetadataDB/Scripts/Script.SetLocalAuthenticationDetails.sql",
"chars": 3123,
"preview": "--Run in SQLCMD mode to use environment variables.\n\n--Optional script used to setup function testing for processing fra"
},
{
"path": "MetadataDB/Security/procfwk.sql",
"chars": 47,
"preview": "CREATE SCHEMA [procfwk]\nAUTHORIZATION [dbo];\n\n"
},
{
"path": "MetadataDB/Security/procfwkHelpers.sql",
"chars": 52,
"preview": "CREATE SCHEMA [procfwkHelpers]\nAUTHORIZATION [dbo];"
},
{
"path": "MetadataDB/Security/procfwkReporting.sql",
"chars": 54,
"preview": "CREATE SCHEMA [procfwkReporting]\nAUTHORIZATION [dbo];"
},
{
"path": "MetadataDB/Security/procfwkTesting.sql",
"chars": 52,
"preview": "CREATE SCHEMA [procfwkTesting]\nAUTHORIZATION [dbo];"
},
{
"path": "MetadataDB/Security/procfwkuser Role.sql",
"chars": 290,
"preview": "/*\nCREATE USER [##Data Factory Name (Managed Identity)##] \nFROM EXTERNAL PROVIDER;\n*/\n\nCREATE ROLE [procfwkuser]\nGO\n\nGR"
},
{
"path": "MetadataDB/dbo/Stored Procedures/DemoModePrecursor.sql",
"chars": 1982,
"preview": "CREATE PROCEDURE [dbo].[DemoModePrecursor]\nAS\nBEGIN\n\n\t--quick win\n\tIF ([procfwk].[GetPropertyValueInternal]('ExecutionP"
},
{
"path": "MetadataDB/dbo/Stored Procedures/ExampleCustomExecutionPrecursor.sql",
"chars": 946,
"preview": "CREATE PROCEDURE [dbo].[ExampleCustomExecutionPrecursor]\nAS\nBEGIN\n\t\n\t--set random Worker pipeline parameter wait times "
},
{
"path": "MetadataDB/dbo/Stored Procedures/FailProcedure.sql",
"chars": 198,
"preview": "CREATE PROCEDURE [dbo].[FailProcedure]\n\t(\n\t@RaiseError VARCHAR(50)\n\t)\nAS\nBEGIN\n\tIF(@RaiseError = 'true')\n\tBEGIN\n\t\tRAISE"
},
{
"path": "MetadataDB/dbo/Tables/ServicePrincipals.sql",
"chars": 372,
"preview": "CREATE TABLE [dbo].[ServicePrincipals](\n\t[CredentialId] INT IDENTITY(1,1) NOT NULL,\n\t[PrincipalName] NVARCHAR(256) NULL"
},
{
"path": "MetadataDB/procfwk/Functions/GetPropertyValueInternal.sql",
"chars": 322,
"preview": "CREATE FUNCTION [procfwk].[GetPropertyValueInternal]\n\t(\n\t@PropertyName VARCHAR(128)\n\t)\nRETURNS NVARCHAR(MAX)\nAS\nBEGIN\n\t"
},
{
"path": "MetadataDB/procfwk/Stored Procedures/BatchWrapper.sql",
"chars": 2751,
"preview": "CREATE PROCEDURE [procfwk].[BatchWrapper]\n\t(\n\t@BatchId UNIQUEIDENTIFIER,\n\t@LocalExecutionId UNIQUEIDENTIFIER OUTPUT\n\t)\n"
},
{
"path": "MetadataDB/procfwk/Stored Procedures/CheckForBlockedPipelines.sql",
"chars": 4174,
"preview": "CREATE PROCEDURE [procfwk].[CheckForBlockedPipelines]\n\t(\n\t@ExecutionId UNIQUEIDENTIFIER,\n\t@StageId INT\n\t)\nAS\nBEGIN\n\tSET"
},
{
"path": "MetadataDB/procfwk/Stored Procedures/CheckForEmailAlerts.sql",
"chars": 1119,
"preview": "CREATE PROCEDURE [procfwk].[CheckForEmailAlerts]\n\t(\n\t@PipelineId INT\n\t)\nAS\nBEGIN\n\tSET NOCOUNT ON;\n\n\tDECLARE @SendAlerts"
},
{
"path": "MetadataDB/procfwk/Stored Procedures/CheckMetadataIntegrity.sql",
"chars": 13313,
"preview": "CREATE PROCEDURE [procfwk].[CheckMetadataIntegrity]\n\t(\n\t@DebugMode BIT = 0,\n\t@BatchName VARCHAR(255) = NULL\n\t)\nAS\nBEGIN\n"
},
{
"path": "MetadataDB/procfwk/Stored Procedures/CheckPreviousExeuction.sql",
"chars": 2887,
"preview": "CREATE PROCEDURE [procfwk].[CheckPreviousExeuction]\n\t(\n\t@BatchName VARCHAR(255) = NULL\n\t)\nAS\nBEGIN\n\tSET NOCOUNT ON;\n\t/*"
},
{
"path": "MetadataDB/procfwk/Stored Procedures/CreateNewExecution.sql",
"chars": 3392,
"preview": "CREATE PROCEDURE [procfwk].[CreateNewExecution]\n\t(\n\t@CallingOrchestratorName NVARCHAR(200),\n\t@LocalExecutionId UNIQUEID"
},
{
"path": "MetadataDB/procfwk/Stored Procedures/ExecutePrecursorProcedure.sql",
"chars": 586,
"preview": "CREATE PROCEDURE [procfwk].[ExecutePrecursorProcedure]\nAS\nBEGIN\n\tDECLARE @SQL VARCHAR(MAX) \n\tDECLARE @ErrorDetail NVARC"
},
{
"path": "MetadataDB/procfwk/Stored Procedures/ExecutionWrapper.sql",
"chars": 3621,
"preview": "CREATE PROCEDURE [procfwk].[ExecutionWrapper]\n\t(\n\t@CallingOrchestratorName NVARCHAR(200) = NULL,\n\t@BatchName VARCHAR(25"
},
{
"path": "MetadataDB/procfwk/Stored Procedures/GetEmailAlertParts.sql",
"chars": 4315,
"preview": "CREATE PROCEDURE [procfwk].[GetEmailAlertParts]\n\t(\n\t@PipelineId INT\n\t)\nAS\nBEGIN\n\tSET NOCOUNT ON;\n\n\tDECLARE @ToRecipient"
},
{
"path": "MetadataDB/procfwk/Stored Procedures/GetFrameworkOrchestratorDetails.sql",
"chars": 795,
"preview": "CREATE PROCEDURE [procfwk].[GetFrameworkOrchestratorDetails]\n\t(\n\t@CallingOrchestratorName NVARCHAR(200)\n\t)\nAS\nBEGIN\n\tSE"
},
{
"path": "MetadataDB/procfwk/Stored Procedures/GetPipelineParameters.sql",
"chars": 1550,
"preview": "CREATE PROCEDURE [procfwk].[GetPipelineParameters]\n\t(\n\t@PipelineId INT\n\t)\nAS\nBEGIN\n\tSET NOCOUNT ON;\n\n\tDECLARE @Json VAR"
},
{
"path": "MetadataDB/procfwk/Stored Procedures/GetPipelinesInStage.sql",
"chars": 366,
"preview": "CREATE PROCEDURE [procfwk].[GetPipelinesInStage]\n\t(\n\t@ExecutionId UNIQUEIDENTIFIER,\n\t@StageId INT\n\t)\nAS\nBEGIN\n\tSET NOCO"
},
{
"path": "MetadataDB/procfwk/Stored Procedures/GetPropertyValue.sql",
"chars": 860,
"preview": "CREATE PROCEDURE [procfwk].[GetPropertyValue]\n\t(\n\t@PropertyName VARCHAR(128)\n\t)\nAS\nBEGIN\t\n\tDECLARE @ErrorDetail NVARCHA"
},
{
"path": "MetadataDB/procfwk/Stored Procedures/GetStages.sql",
"chars": 607,
"preview": "CREATE PROCEDURE [procfwk].[GetStages]\n\t(\n\t@ExecutionId UNIQUEIDENTIFIER\n\t)\nAS\nBEGIN\n\tSET NOCOUNT ON;\n\n\t--defensive che"
},
{
"path": "MetadataDB/procfwk/Stored Procedures/GetWorkerAuthDetails.sql",
"chars": 4579,
"preview": "CREATE PROCEDURE [procfwk].[GetWorkerAuthDetails]\n\t(\n\t@ExecutionId UNIQUEIDENTIFIER,\n\t@StageId INT,\n\t@PipelineId INT\n\t)"
},
{
"path": "MetadataDB/procfwk/Stored Procedures/GetWorkerDetailsWrapper.sql",
"chars": 1535,
"preview": "CREATE PROCEDURE [procfwk].[GetWorkerDetailsWrapper]\n\t(\n\t@ExecutionId UNIQUEIDENTIFIER,\n\t@StageId INT,\n\t@PipelineId INT"
},
{
"path": "MetadataDB/procfwk/Stored Procedures/GetWorkerPipelineDetails.sql",
"chars": 392,
"preview": "CREATE PROCEDURE [procfwk].[GetWorkerPipelineDetails]\n\t(\n\t@ExecutionId UNIQUEIDENTIFIER,\n\t@StageId INT,\n\t@PipelineId IN"
},
{
"path": "MetadataDB/procfwk/Stored Procedures/ResetExecution.sql",
"chars": 3071,
"preview": "CREATE PROCEDURE [procfwk].[ResetExecution]\n\t(\n\t@LocalExecutionId UNIQUEIDENTIFIER = NULL\n\t)\nAS\nBEGIN \n\tSET NOCOUNT\tON;"
},
{
"path": "MetadataDB/procfwk/Stored Procedures/SetErrorLogDetails.sql",
"chars": 942,
"preview": "CREATE PROCEDURE [procfwk].[SetErrorLogDetails]\n\t(\n\t@LocalExecutionId UNIQUEIDENTIFIER,\n\t@JsonErrorDetails VARCHAR(MAX)"
},
{
"path": "MetadataDB/procfwk/Stored Procedures/SetExecutionBlockDependants.sql",
"chars": 455,
"preview": "CREATE PROCEDURE [procfwk].[SetExecutionBlockDependants]\n\t(\n\t@ExecutionId UNIQUEIDENTIFIER = NULL,\n\t@PipelineId INT\n\t)\n"
},
{
"path": "MetadataDB/procfwk/Stored Procedures/SetLogActivityFailed.sql",
"chars": 2484,
"preview": "CREATE PROCEDURE [procfwk].[SetLogActivityFailed]\n\t(\n\t@ExecutionId UNIQUEIDENTIFIER,\n\t@StageId INT,\n\t@PipelineId INT,\n\t"
},
{
"path": "MetadataDB/procfwk/Stored Procedures/SetLogPipelineCancelled.sql",
"chars": 2833,
"preview": "CREATE PROCEDURE [procfwk].[SetLogPipelineCancelled]\n\t(\n\t@ExecutionId UNIQUEIDENTIFIER,\n\t@StageId INT,\n\t@PipelineId INT"
},
{
"path": "MetadataDB/procfwk/Stored Procedures/SetLogPipelineChecking.sql",
"chars": 334,
"preview": "CREATE PROCEDURE [procfwk].[SetLogPipelineChecking]\n\t(\n\t@ExecutionId UNIQUEIDENTIFIER,\n\t@StageId INT,\n\t@PipelineId INT\n"
},
{
"path": "MetadataDB/procfwk/Stored Procedures/SetLogPipelineFailed.sql",
"chars": 2689,
"preview": "CREATE PROCEDURE [procfwk].[SetLogPipelineFailed]\n\t(\n\t@ExecutionId UNIQUEIDENTIFIER,\n\t@StageId INT,\n\t@PipelineId INT,\n\t"
},
{
"path": "MetadataDB/procfwk/Stored Procedures/SetLogPipelineLastStatusCheck.sql",
"chars": 351,
"preview": "CREATE PROCEDURE [procfwk].[SetLogPipelineLastStatusCheck]\n\t(\n\t@ExecutionId UNIQUEIDENTIFIER,\n\t@StageId INT,\n\t@Pipeline"
}
]
// ... and 144 more files (download for full content)
About this extraction
This page contains the full source code of the mrpaulandrew/procfwk GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 344 files (2.6 MB), approximately 705.4k tokens, and a symbol index with 450 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.