Repository: mrpaulandrew/procfwk Branch: master Commit: f51cb458ef7e Files: 344 Total size: 2.6 MB Directory structure: gitextract_8_ebrzfj/ ├── .gitattributes ├── .github/ │ ├── FUNDING.yml │ └── ISSUE_TEMPLATE/ │ ├── bug-found.md │ ├── feature-request.md │ └── help---support-request.md ├── .gitignore ├── .vscode/ │ ├── extensions.json │ ├── launch.json │ ├── settings.json │ └── tasks.json ├── ARM Templates/ │ ├── Data Factory/ │ │ ├── v1.0 Export.json │ │ ├── v1.1 Export.json │ │ ├── v1.2 Export.json │ │ ├── v1.3 Export.json │ │ ├── v1.4 Export.json │ │ ├── v1.5 Export.json │ │ ├── v1.6 Export.json │ │ ├── v1.7 Export.json │ │ ├── v1.8 Export.json │ │ ├── v1.8.3 Export.json │ │ ├── v1.8.5 Export.json │ │ ├── v1.8.6 Export.json │ │ ├── v1.9 Export.json │ │ ├── v1.9.1 Export.json │ │ ├── v1.9.2 Export.json │ │ └── v2.0 Export.json │ ├── Functions App/ │ │ └── v1.6 Export.json │ ├── SQL Database/ │ │ └── v1.6 Export.json │ └── Synapse/ │ └── v2.0 Export.json ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── DataFactory/ │ ├── dataset/ │ │ └── GetSetMetadata.json │ ├── factory/ │ │ └── FrameworkFactory.json │ ├── integrationRuntime/ │ │ └── AzureIR-UKSouth.json │ ├── linkedService/ │ │ ├── FrameworkFunctions.json │ │ ├── Keys.json │ │ └── SupportDatabase.json │ ├── pipeline/ │ │ ├── 01-Grandparent.json │ │ ├── 02-Parent.json │ │ ├── 03-Child.json │ │ ├── 04-Infant.json │ │ ├── Check For Running Pipeline.json │ │ ├── Email Sender.json │ │ ├── Intentional Error.json │ │ ├── Throw Exception.json │ │ ├── Wait 1.json │ │ ├── Wait 10.json │ │ ├── Wait 2.json │ │ ├── Wait 3.json │ │ ├── Wait 4.json │ │ ├── Wait 5.json │ │ ├── Wait 6.json │ │ ├── Wait 7.json │ │ ├── Wait 8.json │ │ └── Wait 9.json │ └── trigger/ │ └── FunctionalTestingTrigger.json ├── DeploymentTools/ │ ├── DataFactory/ │ │ ├── DeployProcFwkComponents.ps1 │ │ ├── DeployProcFwkComponents_Old.ps1 │ │ ├── Get Pipelines.ps1 │ │ ├── GlobalVars.ps1 │ │ ├── PopulatePipelinesInDb.ps1 │ │ ├── ProcFwkComponents.json │ │ └── config-all.csv │ ├── Deployment.targets │ └── DeploymentTools.deployproj ├── FactoryTesting/ │ ├── FactoryTesting.csproj │ ├── Helpers/ │ │ ├── CoverageHelper.cs │ │ ├── DataFactoryHelper.cs │ │ ├── DatabaseHelper.cs │ │ ├── PipelineRunHelper.cs │ │ └── SettingsHelper.cs │ ├── Pipelines/ │ │ ├── 01-Grandparent/ │ │ │ ├── Given300WorkerPipelines.cs │ │ │ ├── Given600WorkerPipelineConcurrentBatches.cs │ │ │ ├── GivenOneWorkerPipeline.cs │ │ │ └── GrandparentHelper.cs │ │ ├── 02-Parent/ │ │ │ ├── Given20ConcurrentBatchesFor1000WorkerPipelines.cs │ │ │ ├── GivenAlreadyRunning.cs │ │ │ ├── GivenBatchExecutionsForConcurrentBatches.cs │ │ │ ├── GivenBatchExecutionsForConcurrentBatchesAlreadyRunning.cs │ │ │ ├── GivenBatchExecutionsForConcurrentBatchesWithSimpleFailureHandling.cs │ │ │ ├── GivenBatchExecutionsForConcurrentBatchesWithSimpleFailureHandlingAndRestart.cs │ │ │ ├── GivenBatchExecutionsForConcurrentBatchesWithSimpleFailureHandlingAndRestartOveride.cs │ │ │ ├── GivenBatchExecutionsForSingleBatch.cs │ │ │ ├── GivenCancelledWorkerAndRestart.cs │ │ │ ├── GivenCancelledWorkerInOneExecutionStage.cs │ │ │ ├── GivenCancelledWorkerThatBlocks.cs │ │ │ ├── GivenCancelledWorkerThatDoesntBlock.cs │ │ │ ├── GivenCleanUpForCancelledWorkerAndRestart.cs │ │ │ ├── GivenCleanUpForSuccessfulWorkersAndRestart.cs │ │ │ ├── GivenDependencyChainFailureHandling.cs │ │ │ ├── GivenDependencyChainFailureHandlingAndRestart.cs │ │ │ ├── GivenDisabledBatches.cs │ │ │ ├── GivenDisabledPipelines.cs │ │ │ ├── GivenDisabledStages.cs │ │ │ ├── GivenNoErrorsAndSPNStoredInDatabase.cs │ │ │ ├── GivenNoErrorsAndSPNStoredInKeyVault.cs │ │ │ ├── GivenNoFailureHandling.cs │ │ │ ├── GivenNoPipelineParameters.cs │ │ │ ├── GivenOneExecutionStage.cs │ │ │ ├── GivenSimpleFailureHandling.cs │ │ │ ├── GivenSimpleFailureHandlingAndRestart.cs │ │ │ ├── GivenSimpleFailureHandlingAndRestartOveride.cs │ │ │ └── ParentHelper.cs │ │ └── Utilities/ │ │ ├── GivenCheckForRunningPipeline.cs │ │ ├── GivenEmailSender.cs │ │ ├── GivenThrowException.cs │ │ └── UtilitiesHelper.cs │ ├── dev.runsettings │ ├── multi.runsettings │ └── test.runsettings ├── Functions/ │ ├── .vscode/ │ │ ├── extensions.json │ │ ├── launch.json │ │ ├── settings.json │ │ └── tasks.json │ ├── Functions/ │ │ ├── CancelPipeline.cs │ │ ├── CheckPipelineStatus.cs │ │ ├── ExecutePipeline.cs │ │ ├── GetActivityErrors.cs │ │ ├── SendEmail.cs │ │ └── ValidatePipeline.cs │ ├── Functions.csproj │ ├── Helpers/ │ │ ├── BodyReader.cs │ │ ├── InvalidRequestException.cs │ │ ├── KeyVaultClient.cs │ │ ├── PipelineRequest.cs │ │ ├── PipelineRunRequest.cs │ │ └── SMTPClient.cs │ ├── Properties/ │ │ └── ServiceDependencies/ │ │ ├── FrameworkSupportFunctions - Zip Deploy/ │ │ │ └── profile.arm.json │ │ └── FrameworkSupportFunctionsBig - Web Deploy/ │ │ └── profile.arm.json │ ├── Services/ │ │ ├── AzureDataFactoryService.cs │ │ ├── AzureSynapseService.cs │ │ ├── PipelineService.cs │ │ ├── PipelineServiceType.cs │ │ └── Returns/ │ │ ├── PipelineDescription.cs │ │ ├── PipelineErrorDetail.cs │ │ └── PipelineRunStatus.cs │ ├── host.json │ └── template_local.settings.json ├── Images/ │ └── procfwk Designs.vsdx ├── LICENSE ├── MetadataDB/ │ ├── MetadataDB.refactorlog │ ├── MetadataDB.sqlproj │ ├── Scripts/ │ │ ├── Alter Database Scale.sql │ │ ├── Handy Selects.sql │ │ ├── LogData/ │ │ │ ├── ErrorLogBackup.sql │ │ │ ├── ErrorLogRestore.sql │ │ │ ├── ExecutionLogBackup.sql │ │ │ └── ExecutionLogRestore.sql │ │ ├── Metadata/ │ │ │ ├── AlertOutcomes.sql │ │ │ ├── DeleteAll.sql │ │ │ ├── DropLegacyObjects.sql │ │ │ ├── DropLegacyTables.sql │ │ │ ├── Orchestrators.sql │ │ │ ├── PipelineDependencies.sql │ │ │ ├── PipelineParams.sql │ │ │ ├── Pipelines.sql │ │ │ ├── Properties.sql │ │ │ ├── RecipientAlertsLink.sql │ │ │ ├── Recipients.sql │ │ │ ├── ReplaceDataFactorys.sql │ │ │ ├── Stages.sql │ │ │ ├── TransferHelperObjects.sql │ │ │ └── TransferReportingObjects.sql │ │ ├── Script.PostDeployment.sql │ │ ├── Script.PreDeployment.sql │ │ └── Script.SetLocalAuthenticationDetails.sql │ ├── Security/ │ │ ├── procfwk.sql │ │ ├── procfwkHelpers.sql │ │ ├── procfwkReporting.sql │ │ ├── procfwkTesting.sql │ │ └── procfwkuser Role.sql │ ├── dbo/ │ │ ├── Stored Procedures/ │ │ │ ├── DemoModePrecursor.sql │ │ │ ├── ExampleCustomExecutionPrecursor.sql │ │ │ └── FailProcedure.sql │ │ └── Tables/ │ │ └── ServicePrincipals.sql │ ├── procfwk/ │ │ ├── Functions/ │ │ │ └── GetPropertyValueInternal.sql │ │ ├── Stored Procedures/ │ │ │ ├── BatchWrapper.sql │ │ │ ├── CheckForBlockedPipelines.sql │ │ │ ├── CheckForEmailAlerts.sql │ │ │ ├── CheckMetadataIntegrity.sql │ │ │ ├── CheckPreviousExeuction.sql │ │ │ ├── CreateNewExecution.sql │ │ │ ├── ExecutePrecursorProcedure.sql │ │ │ ├── ExecutionWrapper.sql │ │ │ ├── GetEmailAlertParts.sql │ │ │ ├── GetFrameworkOrchestratorDetails.sql │ │ │ ├── GetPipelineParameters.sql │ │ │ ├── GetPipelinesInStage.sql │ │ │ ├── GetPropertyValue.sql │ │ │ ├── GetStages.sql │ │ │ ├── GetWorkerAuthDetails.sql │ │ │ ├── GetWorkerDetailsWrapper.sql │ │ │ ├── GetWorkerPipelineDetails.sql │ │ │ ├── ResetExecution.sql │ │ │ ├── SetErrorLogDetails.sql │ │ │ ├── SetExecutionBlockDependants.sql │ │ │ ├── SetLogActivityFailed.sql │ │ │ ├── SetLogPipelineCancelled.sql │ │ │ ├── SetLogPipelineChecking.sql │ │ │ ├── SetLogPipelineFailed.sql │ │ │ ├── SetLogPipelineLastStatusCheck.sql │ │ │ ├── SetLogPipelineRunId.sql │ │ │ ├── SetLogPipelineRunning.sql │ │ │ ├── SetLogPipelineSuccess.sql │ │ │ ├── SetLogPipelineUnknown.sql │ │ │ ├── SetLogPipelineValidating.sql │ │ │ ├── SetLogStagePreparing.sql │ │ │ └── UpdateExecutionLog.sql │ │ ├── Synonyms/ │ │ │ ├── AddPipelineDependant.sql │ │ │ ├── AddProperty.sql │ │ │ ├── AddRecipientPipelineAlerts.sql │ │ │ ├── AddServicePrincipal.sql │ │ │ ├── AddServicePrincipalUrls.sql │ │ │ ├── AddServicePrincipalWrapper.sql │ │ │ ├── AverageStageDuration.sql │ │ │ ├── CheckForValidURL.sql │ │ │ ├── CheckStageAndPiplineIntegrity.sql │ │ │ ├── CompleteExecutionErrorLog.sql │ │ │ ├── CompleteExecutionLog.sql │ │ │ ├── CurrentExecutionSummary.sql │ │ │ ├── DataFactoryDetails.sql │ │ │ ├── DeleteRecipientAlerts.sql │ │ │ ├── DeleteServicePrincipal.sql │ │ │ ├── GetExecutionDetails.sql │ │ │ ├── LastExecution.sql │ │ │ ├── LastExecutionSummary.sql │ │ │ ├── PipelineDependencyChains.sql │ │ │ ├── PipelineProcesses.sql │ │ │ ├── ProcessingStageDetails.sql │ │ │ └── WorkerParallelismOverTime.sql │ │ ├── Tables/ │ │ │ ├── AlertOutcomes.sql │ │ │ ├── BatchExecution.sql │ │ │ ├── BatchStageLink.sql │ │ │ ├── Batches.sql │ │ │ ├── CurrentExecution.sql │ │ │ ├── ErrorLog.sql │ │ │ ├── ExecutionLog.sql │ │ │ ├── Orchestrators.sql │ │ │ ├── PipelineAlertLink.sql │ │ │ ├── PipelineAuthLink.sql │ │ │ ├── PipelineDependencies.sql │ │ │ ├── PipelineParameters.sql │ │ │ ├── Pipelines.sql │ │ │ ├── Properties.sql │ │ │ ├── Recipients.sql │ │ │ ├── Stages.sql │ │ │ ├── Subscriptions.sql │ │ │ └── Tenants.sql │ │ └── Views/ │ │ ├── CurrentProperties.sql │ │ ├── DataFactorys.sql │ │ └── PipelineParameterDataSizes.sql │ ├── procfwkHelpers/ │ │ ├── Functions/ │ │ │ └── CheckForValidURL.sql │ │ ├── Stored Procedures/ │ │ │ ├── AddPipelineDependant.sql │ │ │ ├── AddPipelineViaPowerShell.sql │ │ │ ├── AddProperty.sql │ │ │ ├── AddRecipientPipelineAlerts.sql │ │ │ ├── AddServicePrincipal.sql │ │ │ ├── AddServicePrincipalUrls.sql │ │ │ ├── AddServicePrincipalWrapper.sql │ │ │ ├── CheckStageAndPiplineIntegrity.sql │ │ │ ├── DeleteMetadataWithIntegrity.sql │ │ │ ├── DeleteMetadataWithoutIntegrity.sql │ │ │ ├── DeleteRecipientAlerts.sql │ │ │ ├── DeleteServicePrincipal.sql │ │ │ ├── GetExecutionDetails.sql │ │ │ ├── GetServicePrincipal.sql │ │ │ ├── SetDefaultAlertOutcomes.sql │ │ │ ├── SetDefaultBatchStageLink.sql │ │ │ ├── SetDefaultBatches.sql │ │ │ ├── SetDefaultOrchestrators.sql │ │ │ ├── SetDefaultPipelineDependants.sql │ │ │ ├── SetDefaultPipelineParameters.sql │ │ │ ├── SetDefaultPipelines.sql │ │ │ ├── SetDefaultProperties.sql │ │ │ ├── SetDefaultRecipientPipelineAlerts.sql │ │ │ ├── SetDefaultRecipients.sql │ │ │ ├── SetDefaultStages.sql │ │ │ ├── SetDefaultSubscription.sql │ │ │ └── SetDefaultTenant.sql │ │ └── Views/ │ │ └── PipelineDependencyChains.sql │ ├── procfwkReporting/ │ │ └── Views/ │ │ ├── AverageStageDuration.sql │ │ ├── CompleteExecutionErrorLog.sql │ │ ├── CompleteExecutionLog.sql │ │ ├── CurrentExecutionSummary.sql │ │ ├── LastExecution.sql │ │ ├── LastExecutionSummary.sql │ │ └── WorkerParallelismOverTime.sql │ └── procfwkTesting/ │ └── Stored Procedures/ │ ├── Add20BatchesFor1000Workers.sql │ ├── Add300WorkerPipelineBatches.sql │ ├── Add300WorkerPipelines.sql │ ├── CleanUpMetadata.sql │ ├── GetRunIdWhenAvailable.sql │ └── ResetMetadata.sql ├── MetadataDBTests/ │ ├── MetadataDBTests.sqlproj │ ├── Scripts/ │ │ ├── Script.PostDeployment.sql │ │ └── Script.PreDeployment.sql │ ├── _TestClasses/ │ │ ├── procfwk_GetPropertyValue.sql │ │ └── procfwk_GetPropertyValueInternal.sql │ ├── procfwk_GetPropertyValue/ │ │ ├── setup.sql │ │ ├── test WHEN property does not exist THEN error raised.sql │ │ ├── test WHEN property exists THEN property value returned.sql │ │ ├── test WHEN property invalidated THEN error raised.sql │ │ └── test WHEN property name is null THEN error raised.sql │ ├── procfwk_GetPropertyValueInternal/ │ │ ├── test WHEN property does not exist THEN empty string returned.sql │ │ ├── test WHEN property exists THEN property value returned.sql │ │ └── test WHEN property name is null THEN empty string returned.sql │ └── tSQLt.dacpac ├── Notebooks/ │ ├── Databricks - Throw Exception.scala │ └── Metadata Guide and Handy Code Snippets.ipynb ├── ProcessingFramework.sln ├── README.md ├── Reporting/ │ ├── PowerBI/ │ │ ├── Executions Overview.pbit │ │ └── Executions Overview.pbix │ └── Scoping-Template-LDI-Blank.docx └── Synapse/ ├── dataflow/ │ └── MDF _Order_Count.json ├── dataset/ │ └── GetSetMetadata.json ├── factory/ │ └── FrameworkFactory.json ├── integrationRuntime/ │ └── AutoResolveIntegrationRuntime.json ├── linkedService/ │ ├── FrameworkFunctions.json │ ├── Keys.json │ ├── SupportDatabase.json │ ├── covid-tracking.json │ ├── procfwkforsynapse-WorkspaceDefaultSqlServer.json │ └── procfwkforsynapse-WorkspaceDefaultStorage.json ├── notebook/ │ └── Getting Started with Delta Lake.json ├── pipeline/ │ ├── 01-Grandparent.json │ ├── 02-Parent.json │ ├── 03-Child.json │ ├── 04-Infant.json │ ├── Check For Running Pipeline.json │ ├── Email Sender.json │ ├── Intentional Error.json │ ├── Throw Exception.json │ ├── Wait 1.json │ ├── Wait 10.json │ ├── Wait 2.json │ ├── Wait 3.json │ ├── Wait 4.json │ ├── Wait 5.json │ ├── Wait 6.json │ ├── Wait 7.json │ ├── Wait 8.json │ └── Wait 9.json └── trigger/ └── FunctionalTestingTrigger.json ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitattributes ================================================ # Auto detect text files and perform LF normalization * text=auto ================================================ FILE: .github/FUNDING.yml ================================================ # These are supported funding model platforms github: [mrpaulandrew] ================================================ FILE: .github/ISSUE_TEMPLATE/bug-found.md ================================================ --- name: Bug Found about: Tell me about the bug you've found in the procfwk please title: '' labels: bug assignees: mrpaulandrew --- **Describe the bug** A clear and concise description of what the bug is. **Affected services** Which resource within the processing framework does this affect? * Data Factory/Synapse * SQL Database * Functions * All of them * Other **To Reproduce** Steps to reproduce the behavior: 1. Go to '...' 2. Click on '....' 3. Scroll down to '....' 4. See error **Expected behaviour** A clear and concise description of what you expected to happen. **Screenshots** If applicable, add screenshots to help explain your problem. **Additional context** Add any other context about the problem here. ================================================ FILE: .github/ISSUE_TEMPLATE/feature-request.md ================================================ --- name: Feature Request about: Tell me about your idea to enhance the procfwk please title: '' labels: enhancement assignees: mrpaulandrew --- **Share Your Idea** All features considered. ================================================ FILE: .github/ISSUE_TEMPLATE/help---support-request.md ================================================ --- name: Help & Support Request about: Tell me about the problem or error you are facing when using the procfwk title: '' labels: help wanted assignees: mrpaulandrew --- **Describe the error** A clear and concise description of what the bug is. **Error message** An output of the error message presented. **Affected services** Which resource within the processing framework does this affect? * Data Factory/Synapse * SQL Database * Functions * All of them * Other **To Reproduce** Steps to reproduce the behavior: 1. Go to '...' 2. Click on '....' 3. Scroll down to '....' 4. See error **Screenshots** If applicable, add screenshots to help explain your problem. ================================================ FILE: .gitignore ================================================ ################################################################################ # This .gitignore file was automatically created by Microsoft(R) Visual Studio. ################################################################################ ## Ignore Visual Studio temporary files, build results, and ## files generated by popular Visual Studio add-ons. ## https://stackoverflow.com/questions/30868544/gitignore-wont-ignore-vs-folder-for-visual-studio-2015-rc-on-windows7-8/39520183 # User-specific files *.suo *.user *.userosscache *.sln.docstates # User-specific files (MonoDevelop/Xamarin Studio) *.userprefs # Build results [Dd]ebug/ [Dd]ebugPublic/ [Rr]elease/ [Rr]eleases/ x64/ x86/ bld/ [Bb]in/ [Oo]bj/ [Ll]og/ # Visual Studio 2015 cache/options directory .vs/ # Uncomment if you have tasks that create the project's static files in wwwroot #wwwroot/ # MSTest test Results [Tt]est[Rr]esult*/ [Bb]uild[Ll]og.* # NUNIT *.VisualState.xml TestResult.xml # Build Results of an ATL Project [Dd]ebugPS/ [Rr]eleasePS/ dlldata.c # DNX project.lock.json artifacts/ *_i.c *_p.c *_i.h *.ilk *.meta *.obj *.pch *.pdb *.pgc *.pgd *.rsp *.sbr *.tlb *.tli *.tlh *.tmp *.tmp_proj *.log *.vspscc *.vssscc .builds *.pidb *.svclog *.scc # Chutzpah Test files _Chutzpah* # Visual C++ cache files ipch/ *.aps *.ncb *.opendb *.opensdf *.sdf *.cachefile # Visual Studio profiler *.psess *.vsp *.vspx *.sap # TFS 2012 Local Workspace $tf/ # Guidance Automation Toolkit *.gpState # ReSharper is a .NET coding add-in _ReSharper*/ *.[Rr]e[Ss]harper *.DotSettings.user # JustCode is a .NET coding add-in .JustCode # TeamCity is a build add-in _TeamCity* # DotCover is a Code Coverage Tool *.dotCover # NCrunch _NCrunch_* .*crunch*.local.xml nCrunchTemp_* # MightyMoose *.mm.* AutoTest.Net/ # Web workbench (sass) .sass-cache/ # Installshield output folder [Ee]xpress/ # DocProject is a documentation generator add-in DocProject/buildhelp/ DocProject/Help/*.HxT DocProject/Help/*.HxC DocProject/Help/*.hhc DocProject/Help/*.hhk DocProject/Help/*.hhp DocProject/Help/Html2 DocProject/Help/html # Click-Once directory publish/ # Publish Web Output *.[Pp]ublish.xml *.azurePubxml # TODO: Comment the next line if you want to checkin your web deploy settings # but database connection strings (with potential passwords) will be unencrypted *.pubxml *.publishproj # NuGet Packages *.nupkg # The packages folder can be ignored because of Package Restore **/packages/* # except build/, which is used as an MSBuild target. !**/packages/build/ # Uncomment if necessary however generally it will be regenerated when needed #!**/packages/repositories.config # NuGet v3's project.json files produces more ignoreable files *.nuget.props *.nuget.targets # Microsoft Azure Build Output csx/ *.build.csdef # Microsoft Azure Emulator ecf/ rcf/ # Windows Store app package directories and files AppPackages/ BundleArtifacts/ Package.StoreAssociation.xml _pkginfo.txt # Visual Studio cache files # files ending in .cache can be ignored *.[Cc]ache # but keep track of directories ending in .cache !*.[Cc]ache/ # Others ClientBin/ ~$* *~ *.dbmdl *.dbproj.schemaview *.pfx *.publishsettings node_modules/ orleans.codegen.cs # Since there are multiple workflows, uncomment next line to ignore bower_components # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) #bower_components/ # RIA/Silverlight projects Generated_Code/ # Backup & report files from converting an old project file # to a newer Visual Studio version. Backup files are not needed, # because we have git ;-) _UpgradeReport_Files/ Backup*/ UpgradeLog*.XML UpgradeLog*.htm # SQL Server files *.mdf *.ldf *.jfm # Business Intelligence projects *.rdl.data *.bim.layout *.bim_*.settings # Microsoft Fakes FakesAssemblies/ # GhostDoc plugin setting file *.GhostDoc.xml # Node.js Tools for Visual Studio .ntvs_analysis.dat # Visual Studio 6 build log *.plg # Visual Studio 6 workspace options file *.opt # Visual Studio LightSwitch build output **/*.HTMLClient/GeneratedArtifacts **/*.DesktopClient/GeneratedArtifacts **/*.DesktopClient/ModelManifest.xml **/*.Server/GeneratedArtifacts **/*.Server/ModelManifest.xml _Pvt_Extensions # Paket dependency manager .paket/paket.exe paket-files/ # FAKE - F# Make .fake/ # JetBrains Rider .idea/ *.sln.iml /PipelineExecutor/local.settings.json /Functions/local.settings.json /docs ================================================ FILE: .vscode/extensions.json ================================================ { "recommendations": [ "ms-azuretools.vscode-azurefunctions", "ms-vscode.csharp" ] } ================================================ FILE: .vscode/launch.json ================================================ { "version": "0.2.0", "configurations": [ { "name": "Attach to .NET Functions", "type": "coreclr", "request": "attach", "processId": "${command:azureFunctions.pickProcess}" } ] } ================================================ FILE: .vscode/settings.json ================================================ { "azureFunctions.deploySubpath": "PipelineExecutor/bin/Release/netcoreapp3.1/publish", "azureFunctions.projectLanguage": "C#", "azureFunctions.projectRuntime": "~3", "debug.internalConsoleOptions": "neverOpen", "azureFunctions.preDeployTask": "publish" } ================================================ FILE: .vscode/tasks.json ================================================ { "version": "2.0.0", "tasks": [ { "label": "clean", "command": "dotnet", "args": [ "clean", "/property:GenerateFullPaths=true", "/consoleloggerparameters:NoSummary" ], "type": "process", "problemMatcher": "$msCompile", "options": { "cwd": "${workspaceFolder}/PipelineExecutor" } }, { "label": "build", "command": "dotnet", "args": [ "build", "/property:GenerateFullPaths=true", "/consoleloggerparameters:NoSummary" ], "type": "process", "dependsOn": "clean", "group": { "kind": "build", "isDefault": true }, "problemMatcher": "$msCompile", "options": { "cwd": "${workspaceFolder}/PipelineExecutor" } }, { "label": "clean release", "command": "dotnet", "args": [ "clean", "--configuration", "Release", "/property:GenerateFullPaths=true", "/consoleloggerparameters:NoSummary" ], "type": "process", "problemMatcher": "$msCompile", "options": { "cwd": "${workspaceFolder}/PipelineExecutor" } }, { "label": "publish", "command": "dotnet", "args": [ "publish", "--configuration", "Release", "/property:GenerateFullPaths=true", "/consoleloggerparameters:NoSummary" ], "type": "process", "dependsOn": "clean release", "problemMatcher": "$msCompile", "options": { "cwd": "${workspaceFolder}/PipelineExecutor" } }, { "type": "func", "dependsOn": "build", "options": { "cwd": "${workspaceFolder}/PipelineExecutor/bin/Debug/netcoreapp3.1" }, "command": "host start", "isBackground": true, "problemMatcher": "$func-watch" } ] } ================================================ FILE: ARM Templates/Data Factory/v1.0 Export.json ================================================ { "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { "factoryName": { "type": "string", "metadata": "Data Factory name" }, "Keys_properties_typeProperties_baseUrl": { "type": "string" }, "SupportDatabase_properties_typeProperties_connectionString_secretName": { "type": "string" } }, "variables": { "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" }, "resources": [ { "name": "[concat(parameters('factoryName'), '/BootStrap')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Get Stages", "type": "Lookup", "dependsOn": [ { "activity": "Create New Execution", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetProcessStages]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@activity('Create New Execution').output.firstRow.ExecutionId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Stages", "type": "ForEach", "dependsOn": [ { "activity": "Get Stages", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Stages').output.value", "type": "Expression" }, "isSequential": true, "activities": [ { "name": "Stage Executor", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Log Stage Start", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "Executor", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "StageId": { "value": "@item().StageId", "type": "Expression" }, "ExecutionId": { "value": "@activity('Create New Execution').output.firstRow.ExecutionId", "type": "Expression" } } } }, { "name": "Log Stage Start", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogStageStart]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@activity('Create New Execution').output.firstRow.ExecutionId", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Create New Execution", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[CreateNewExecution]", "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Update Execution Log", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Stages", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[UpdateExecutionLog]" }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ], "folder": { "name": "Framework" }, "annotations": [] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", "[concat(variables('factoryId'), '/pipelines/Executor')]" ] }, { "name": "[concat(parameters('factoryName'), '/Child 1')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Wait1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 1 } } ], "folder": { "name": "Child Pipelines" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Child 2')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Wait1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 1 } } ], "folder": { "name": "Child Pipelines" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Child 3')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Wait1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 1 } } ], "folder": { "name": "Child Pipelines" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Child 4')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Wait1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 1 } } ], "folder": { "name": "Child Pipelines" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Child 5')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Wait1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 1 } } ], "folder": { "name": "Child Pipelines" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Child 6')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Wait1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 1 } } ], "folder": { "name": "Child Pipelines" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Executor')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Get Pipelines", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelinesInStage]", "storedProcedureParameters": { "StageId": { "type": "Int32", "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Pipelines", "type": "ForEach", "dependsOn": [ { "activity": "Get Pipelines", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Pipelines').output.value", "type": "Expression" }, "activities": [ { "name": "Execute Pipeline", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Set Body", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Log Pipeline Running", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "ExecutePipeline", "method": "POST", "headers": {}, "body": { "value": "@variables('FunctionBody')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "PipelineExecutor", "type": "LinkedServiceReference" } }, { "name": "Set Body", "type": "SetVariable", "dependsOn": [ { "activity": "Get Pipeline Params", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "FunctionBody", "value": { "value": "@concat('\n{\n\t\"tenantId\": \"1234-1234-1234-1234-1234\",\n\t\"applicationId\": \"1234-1234-1234-1234-1234\",\n\t\"authenticationKey\": \"Passw0rd123!\",\n\t\"subscriptionId\": \"1234-1234-1234-1234-1234\",\n\t\"resourceGroup\": \"Demos\",\n\t\"factoryName\": \"FunFactory\",\n\t\"pipelineName\": \"',item().PipelineName,'\"',activity('Get Pipeline Params').output.firstRow.Params,'\n}')", "type": "Expression" } } }, { "name": "Get Pipeline Params", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelineParameters]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@item().PipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Pipeline Running", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Log Pipeline Success", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Log Pipeline Failure", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } } ], "parameters": { "StageId": { "type": "int" }, "ExecutionId": { "type": "string" } }, "variables": { "FunctionBody": { "type": "String" } }, "folder": { "name": "Framework" }, "annotations": [] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/PipelineExecutor')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/Grandparent')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Framework Processing", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "BootStrap", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": {} } } ], "annotations": [] }, "dependsOn": [ "[concat(variables('factoryId'), '/pipelines/BootStrap')]" ] }, { "name": "[concat(parameters('factoryName'), '/GetSetMetadata')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" }, "annotations": [], "type": "AzureSqlTable", "schema": [], "typeProperties": {} }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/Keys')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [], "type": "AzureKeyVault", "typeProperties": { "baseUrl": "[parameters('Keys_properties_typeProperties_baseUrl')]" } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/PipelineExecutor')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [], "type": "AzureFunction", "typeProperties": { "functionAppUrl": "https://pipelineexecutor.azurewebsites.net", "functionKey": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "ExecutorFunctionAppKey" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/SupportDatabase')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [], "type": "AzureSqlDatabase", "typeProperties": { "connectionString": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "[parameters('SupportDatabase_properties_typeProperties_connectionString_secretName')]" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] } ] } ================================================ FILE: ARM Templates/Data Factory/v1.1 Export.json ================================================ { "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { "factoryName": { "type": "string", "metadata": "Data Factory name", "defaultValue": "" }, "Keys_properties_typeProperties_baseUrl": { "type": "string", "defaultValue": "" }, "SupportDatabase_properties_typeProperties_connectionString_secretName": { "type": "string", "defaultValue": "" } }, "variables": { "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" }, "resources": [ { "name": "[concat(parameters('factoryName'), '/BootStrap')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Get Stages", "type": "Lookup", "dependsOn": [ { "activity": "Create New Execution", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetProcessStages]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@activity('Create New Execution').output.firstRow.ExecutionId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Stages", "type": "ForEach", "dependsOn": [ { "activity": "Get Stages", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Tenant Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Subscription Id", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Stages').output.value", "type": "Expression" }, "isSequential": true, "activities": [ { "name": "Stage Executor", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Log Stage Start", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "Executor", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "StageId": { "value": "@item().StageId", "type": "Expression" }, "ExecutionId": { "value": "@activity('Create New Execution').output.firstRow.ExecutionId", "type": "Expression" }, "TenantId": { "value": "@activity('Get Tenant Id').output.firstRow.PropertyValue", "type": "Expression" }, "SubscriptionId": { "value": "@activity('Get Subscription Id').output.firstRow.PropertyValue", "type": "Expression" } } } }, { "name": "Log Stage Start", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogStageStart]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@activity('Create New Execution').output.firstRow.ExecutionId", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Create New Execution", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[CreateNewExecution]", "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Update Execution Log", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Stages", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[UpdateExecutionLog]" }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Tenant Id", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "TenantId" } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Get Subscription Id", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "SubscriptionId" } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } } ], "folder": { "name": "Framework" }, "annotations": [] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", "[concat(variables('factoryId'), '/pipelines/Executor')]" ] }, { "name": "[concat(parameters('factoryName'), '/Child 1')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Wait1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 1 } } ], "folder": { "name": "Child Pipelines" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Child 2')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Wait1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 1 } } ], "folder": { "name": "Child Pipelines" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Child 3')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Wait1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 1 } } ], "folder": { "name": "Child Pipelines" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Child 4')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Wait1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 1 } } ], "folder": { "name": "Child Pipelines" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Child 5')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Wait1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 1 } } ], "folder": { "name": "Child Pipelines" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Child 6')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Wait1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 1 } } ], "folder": { "name": "Child Pipelines" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Executor')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Get Pipelines", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelinesInStage]", "storedProcedureParameters": { "StageId": { "type": "Int32", "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Pipelines", "type": "ForEach", "dependsOn": [ { "activity": "Get Pipelines", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Pipelines').output.value", "type": "Expression" }, "activities": [ { "name": "Execute Pipeline", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Log Pipeline Running", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Pipeline Params", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get SPN Details", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "ExecutePipeline", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n\t\"tenantId\": \"',pipeline().parameters.TenantId,'\",\n\t\"applicationId\": \"',activity('Get SPN Details').output.firstRow.Id,'\",\n\t\"authenticationKey\": \"',activity('Get SPN Details').output.firstRow.Secret,'\",\n\t\"subscriptionId\": \"',pipeline().parameters.SubscriptionId,'\",\n\t\"resourceGroup\": \"',item().ResourceGroupName,'\",\n\t\"factoryName\": \"',item().DataFactoryName,'\",\n\t\"pipelineName\": \"',item().PipelineName,'\"',activity('Get Pipeline Params').output.firstRow.Params,'\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "PipelineExecutor", "type": "LinkedServiceReference" } }, { "name": "Get Pipeline Params", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelineParameters]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@item().PipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Pipeline Running", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Log Pipeline Success", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Log Pipeline Failure", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get SPN Details", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetServicePrincipal]", "storedProcedureParameters": { "DataFactory": { "type": "String", "value": { "value": "@item().DataFactoryName", "type": "Expression" } }, "PipelineName": { "type": "String", "value": { "value": "@item().PipelineName", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } } ] } } ], "parameters": { "StageId": { "type": "int" }, "ExecutionId": { "type": "string" }, "TenantId": { "type": "string" }, "SubscriptionId": { "type": "string" } }, "variables": { "FunctionBody": { "type": "String" } }, "folder": { "name": "Framework" }, "annotations": [] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/PipelineExecutor')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/Grandparent')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Framework Processing", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "BootStrap", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": {} } } ], "annotations": [] }, "dependsOn": [ "[concat(variables('factoryId'), '/pipelines/BootStrap')]" ] }, { "name": "[concat(parameters('factoryName'), '/GetSetMetadata')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" }, "annotations": [], "type": "AzureSqlTable", "schema": [], "typeProperties": {} }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/Keys')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [], "type": "AzureKeyVault", "typeProperties": { "baseUrl": "[parameters('Keys_properties_typeProperties_baseUrl')]" } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/PipelineExecutor')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [], "type": "AzureFunction", "typeProperties": { "functionAppUrl": "https://pipelineexecutor.azurewebsites.net", "functionKey": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "ExecutorFunctionAppKey" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/SupportDatabase')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [], "type": "AzureSqlDatabase", "typeProperties": { "connectionString": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "[parameters('SupportDatabase_properties_typeProperties_connectionString_secretName')]" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] } ] } ================================================ FILE: ARM Templates/Data Factory/v1.2 Export.json ================================================ { "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { "factoryName": { "type": "string", "metadata": "Data Factory name", "defaultValue": "" }, "Keys_properties_typeProperties_baseUrl": { "type": "string", "defaultValue": "" }, "SupportDatabase_properties_typeProperties_connectionString_secretName": { "type": "string", "defaultValue": "" } }, "variables": { "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" }, "resources": [ { "name": "[concat(parameters('factoryName'), '/01 Grandparent')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Framework Processing", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "02 Parent", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": {} } } ], "folder": { "name": "_ProcFwk" }, "annotations": [] }, "dependsOn": [ "[concat(variables('factoryId'), '/pipelines/02 Parent')]" ] }, { "name": "[concat(parameters('factoryName'), '/02 Parent')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk parent pipeline used to bootstrap the orchestration framework in perform the first level ForEach calls in sequence for the metadata stages.", "activities": [ { "name": "Get Stages", "description": "Returns a distinct list of execution stages within the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Execution Wrapper", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetStages]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Stages", "description": "Top level ForEach to sequentially call all processing stages within the framework metadata. Items for iteration passed from the Get Stages lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Stages", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Tenant Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Subscription Id", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Stages').output.value", "type": "Expression" }, "isSequential": true, "activities": [ { "name": "Stage Executor", "description": "Call to the framework generic child pipeline for a given execution stage.", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Log Stage Preparing", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "03 Child", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "StageId": { "value": "@item().StageId", "type": "Expression" }, "ExecutionId": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" }, "TenantId": { "value": "@activity('Get Tenant Id').output.firstRow.PropertyValue", "type": "Expression" }, "SubscriptionId": { "value": "@activity('Get Subscription Id').output.firstRow.PropertyValue", "type": "Expression" } } } }, { "name": "Log Stage Preparing", "description": "Update the current execution table flagging all pipelines within the stage as preparing.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Check for Blockers", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogStagePreparing]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check for Blockers", "description": "Used to double check and stop the next execution stage if failures and blockers have be incurred. Without this step processing would continue regardless of upstream failures.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[CheckForBlockedPipelines]", "storedProcedureParameters": { "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Execution Wrapper", "description": "Wrapper to reset and restart processing or create a completely new execution instance of the framework metadata.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[ExecutionWrapper]", "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Archive Execution Log", "description": "After a successful execution run the current execution metadata is moved to the long term logging table by this stored procedure call.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Stages", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[UpdateExecutionLog]" }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Tenant Id", "description": "Returning the Azure Tenant Id from the metadata properties table.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "TenantId" } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Get Subscription Id", "description": "Returning the Azure Subscription Id from the metadata properties table.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "SubscriptionId" } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } } ], "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", "[concat(variables('factoryId'), '/pipelines/03 Child')]" ] }, { "name": "[concat(parameters('factoryName'), '/03 Child')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk child pipeline used to execute processing pipelines within a given execution stage. This pipeline will be called multi times in parallel.", "activities": [ { "name": "Get Pipelines", "description": "Returns all pipelines from the metadata to be executed within a given processing stage.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelinesInStage]", "storedProcedureParameters": { "StageId": { "type": "Int32", "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Pipelines", "description": "Second level ForEach to run in parallel all pipelines within the stage. Items for iteration passed from the Get Pipelines lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Pipelines", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Pipelines').output.value", "type": "Expression" }, "activities": [ { "name": "Execute Pipeline", "description": "The lowest level executor with the metadata framework to call existing processing pipelines within Data Factory. The function called will block processing and wait for an outcome.", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Log Pipeline Running", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Pipeline Params", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get SPN Details", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "ExecutePipeline", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n\t\"tenantId\": \"',pipeline().parameters.TenantId,'\",\n\t\"applicationId\": \"',activity('Get SPN Details').output.firstRow.Id,'\",\n\t\"authenticationKey\": \"',activity('Get SPN Details').output.firstRow.Secret,'\",\n\t\"subscriptionId\": \"',pipeline().parameters.SubscriptionId,'\",\n\t\"resourceGroup\": \"',item().ResourceGroupName,'\",\n\t\"factoryName\": \"',item().DataFactoryName,'\",\n\t\"pipelineName\": \"',item().PipelineName,'\"',activity('Get Pipeline Params').output.firstRow.Params,'\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "PipelineExecutor", "type": "LinkedServiceReference" } }, { "name": "Get Pipeline Params", "description": "Returns any parameters from metadata required for the processing pipeline being called. The output can be an empty string if no parameters are required.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelineParameters]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@item().PipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Pipeline Running", "description": "Sets the current pipeline with a status of running within the current execution database table.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get SPN Details", "description": "Return the SPN ID and Secret for the processing pipeline being executed. Called at this level as each pipeline can have a different SPN.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetServicePrincipal]", "storedProcedureParameters": { "DataFactory": { "type": "String", "value": { "value": "@item().DataFactoryName", "type": "Expression" } }, "PipelineName": { "type": "String", "value": { "value": "@item().PipelineName", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Handle Function Output", "description": "Receives the outcome from the function execution for a given processing pipeline and updates the current execution table with different pipelines status values depending on the result (case).", "type": "Switch", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@activity('Execute Pipeline').output.Status", "type": "Expression" }, "cases": [ { "value": "Failed", "activities": [ { "name": "Log Pipeline Failure", "description": "Updates the current execution table with a pipeline status of failed if the function outcome is failed. Also blocks pipelines in the downstream execution stage.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@activity('Execute Pipeline').output.RunIdUsed", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Succeeded", "activities": [ { "name": "Log Pipeline Success", "description": "Updates the current execution table with a pipeline status of success if the function outcome is succeeded.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId\n", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } ], "defaultActivities": [ { "name": "Log Pipeline Unknown", "description": "Updates the current execution table with a pipeline status of unknown if the function returns an unexpected outcome.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineUnknown]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Log Activity Failure", "description": "Handle true failures from calling out to the Azure Function and update the current execution table accordingly so a restart can occur.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@pipeline().RunId", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } } ], "parameters": { "StageId": { "type": "int" }, "ExecutionId": { "type": "string" }, "TenantId": { "type": "string" }, "SubscriptionId": { "type": "string" } }, "variables": { "FunctionBody": { "type": "String" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/PipelineExecutor')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/Intention Error')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Call Fail Procedure", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[dbo].[FailProcedure]" }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ], "folder": { "name": "Process Pipelines" }, "annotations": [] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/Wait 1')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "Process Pipelines" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 10')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait10", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "Process Pipelines" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 2')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait2", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "Process Pipelines" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 3')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait3", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "Process Pipelines" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 4')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait4", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "Process Pipelines" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 5')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait5", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "Process Pipelines" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 6')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait6", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "Process Pipelines" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 7')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait7", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "Process Pipelines" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 8')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait8", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "Process Pipelines" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 9')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait9", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "Process Pipelines" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/GetSetMetadata')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" }, "annotations": [ "ADF.procfwk" ], "type": "AzureSqlTable", "schema": [], "typeProperties": {} }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/Keys')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection to Key Vault for all other ADF linked service credentials required to run the processing framework.", "annotations": [ "ADF.procfwk" ], "type": "AzureKeyVault", "typeProperties": { "baseUrl": "[parameters('Keys_properties_typeProperties_baseUrl')]" } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/PipelineExecutor')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection to the Function App from ADF for calling the processing pipeline function within the orchestration framework.", "annotations": [ "ADF.procfwk" ], "type": "AzureFunction", "typeProperties": { "functionAppUrl": "https://pipelineexecutor.azurewebsites.net", "functionKey": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "ExecutorFunctionAppKey" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/SupportDatabase')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection between ADF and processing framework metadata SQLDB.", "annotations": [ "ADF.procfwk" ], "type": "AzureSqlDatabase", "typeProperties": { "connectionString": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "[parameters('SupportDatabase_properties_typeProperties_connectionString_secretName')]" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] } ] } ================================================ FILE: ARM Templates/Data Factory/v1.3 Export.json ================================================ { "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { "factoryName": { "type": "string", "metadata": "Data Factory name", "defaultValue": "" }, "Keys_properties_typeProperties_baseUrl": { "type": "string", "defaultValue": "" }, "SupportDatabase_properties_typeProperties_connectionString_secretName": { "type": "string", "defaultValue": "" } }, "variables": { "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" }, "resources": [ { "name": "[concat(parameters('factoryName'), '/01-Grandparent')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk grandparent pipeline used optionally to bootstrap any wider processes in your Data Factory that then calls the processing framework.", "activities": [ { "name": "Framework Processing", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Set Random Waits", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "02-Parent", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": {} } }, { "name": "Set Random Waits", "description": "For functional testing only.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[dbo].[SetRandomWaitValues]" }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ], "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/pipelines/02-Parent')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/02-Parent')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk parent pipeline used to bootstrap the orchestration framework in perform the first level ForEach calls in sequence for the metadata stages.", "activities": [ { "name": "Get Stages", "description": "Returns a distinct list of execution stages within the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Execution Wrapper", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetStages]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Stages", "description": "Top level ForEach to sequentially call all processing stages within the framework metadata. Items for iteration passed from the Get Stages lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Stages", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Tenant Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Subscription Id", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Stages').output.value", "type": "Expression" }, "isSequential": true, "activities": [ { "name": "Stage Executor", "description": "Call to the framework generic child pipeline for a given execution stage.", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Log Stage Preparing", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "03-Child", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "StageId": { "value": "@item().StageId", "type": "Expression" }, "ExecutionId": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" }, "TenantId": { "value": "@activity('Get Tenant Id').output.firstRow.PropertyValue", "type": "Expression" }, "SubscriptionId": { "value": "@activity('Get Subscription Id').output.firstRow.PropertyValue", "type": "Expression" } } } }, { "name": "Log Stage Preparing", "description": "Update the current execution table flagging all pipelines within the stage as preparing.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Check for Blockers", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogStagePreparing]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check for Blockers", "description": "Used to double check and stop the next execution stage if failures and blockers have be incurred. Without this step processing would continue regardless of upstream failures.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[CheckForBlockedPipelines]", "storedProcedureParameters": { "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Execution Wrapper", "description": "Wrapper to reset and restart processing or create a completely new execution instance of the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Metadata Integrity Checks", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[ExecutionWrapper]", "storedProcedureParameters": { "CallingDataFactory": { "type": "String", "value": { "value": "@pipeline().DataFactory", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Archive Execution Log", "description": "After a successful execution run the current execution metadata is moved to the long term logging table by this stored procedure call.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Stages", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[UpdateExecutionLog]" }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Tenant Id", "description": "Returning the Azure Tenant Id from the metadata properties table.", "type": "Lookup", "dependsOn": [ { "activity": "Metadata Integrity Checks", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "TenantId" } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Get Subscription Id", "description": "Returning the Azure Subscription Id from the metadata properties table.", "type": "Lookup", "dependsOn": [ { "activity": "Metadata Integrity Checks", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "SubscriptionId" } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Metadata Integrity Checks", "description": "Performs a series of checks on all metadata held in the framework SQLDB. This is intended to raise errors before an execution run even starts.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[CheckMetadataIntegrity]", "storedProcedureParameters": { "DebugMode": { "value": "false", "type": "Boolean" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ], "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", "[concat(variables('factoryId'), '/pipelines/03-Child')]" ] }, { "name": "[concat(parameters('factoryName'), '/03-Child')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk child pipeline used to execute processing pipelines within a given execution stage. This pipeline will be called multi times in parallel.", "activities": [ { "name": "Get Pipelines", "description": "Returns all pipelines from the metadata to be executed within a given processing stage.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelinesInStage]", "storedProcedureParameters": { "StageId": { "type": "Int32", "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Pipelines", "description": "Second level ForEach to run in parallel all pipelines within the stage. Items for iteration passed from the Get Pipelines lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Pipelines", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Pipelines').output.value", "type": "Expression" }, "activities": [ { "name": "Execute Pipeline", "description": "The lowest level executor with the metadata framework to call existing processing pipelines within Data Factory. The function called will block processing and wait for an outcome.", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Log Pipeline Running", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Pipeline Params", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get SPN Details", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "ExecutePipeline", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n\t\"tenantId\": \"',pipeline().parameters.TenantId,'\",\n\t\"applicationId\": \"',activity('Get SPN Details').output.firstRow.Id,'\",\n\t\"authenticationKey\": \"',activity('Get SPN Details').output.firstRow.Secret,'\",\n\t\"subscriptionId\": \"',pipeline().parameters.SubscriptionId,'\",\n\t\"resourceGroup\": \"',item().ResourceGroupName,'\",\n\t\"factoryName\": \"',item().DataFactoryName,'\",\n\t\"pipelineName\": \"',item().PipelineName,'\"',activity('Get Pipeline Params').output.firstRow.Params,'\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "PipelineExecutor", "type": "LinkedServiceReference" } }, { "name": "Get Pipeline Params", "description": "Returns any parameters from metadata required for the processing pipeline being called. The output can be an empty string if no parameters are required.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelineParameters]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@item().PipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Pipeline Running", "description": "Sets the current pipeline with a status of running within the current execution database table.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get SPN Details", "description": "Return the SPN ID and Secret for the processing pipeline being executed. Called at this level as each pipeline can have a different SPN.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetServicePrincipal]", "storedProcedureParameters": { "DataFactory": { "type": "String", "value": { "value": "@item().DataFactoryName", "type": "Expression" } }, "PipelineName": { "type": "String", "value": { "value": "@item().PipelineName", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Handle Function Output", "description": "Receives the outcome from the function execution for a given processing pipeline and updates the current execution table with different pipelines status values depending on the result (case).", "type": "Switch", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@activity('Execute Pipeline').output.Status", "type": "Expression" }, "cases": [ { "value": "Failed", "activities": [ { "name": "Log Pipeline Failure", "description": "Updates the current execution table with a pipeline status of failed if the function outcome is failed. Also blocks pipelines in the downstream execution stage.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@activity('Execute Pipeline').output.RunIdUsed", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Succeeded", "activities": [ { "name": "Log Pipeline Success", "description": "Updates the current execution table with a pipeline status of success if the function outcome is succeeded.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId\n", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } ], "defaultActivities": [ { "name": "Log Pipeline Unknown", "description": "Updates the current execution table with a pipeline status of unknown if the function returns an unexpected outcome.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineUnknown]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Log Activity Failure", "description": "Handle true failures from calling out to the Azure Function and update the current execution table accordingly so a restart can occur.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@pipeline().RunId", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } } ], "parameters": { "StageId": { "type": "int" }, "ExecutionId": { "type": "string" }, "TenantId": { "type": "string" }, "SubscriptionId": { "type": "string" } }, "variables": { "FunctionBody": { "type": "String" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/PipelineExecutor')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/Intention Error')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Call Fail Procedure", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[dbo].[FailProcedure]" }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ], "folder": { "name": "Processes" }, "annotations": [] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/Wait 1')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "Processes" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 10')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait10", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "Processes" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 2')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait2", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "Processes" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 3')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait3", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "Processes" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 4')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait4", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "Processes" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 5')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait5", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "Processes" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 6')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait6", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "Processes" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 7')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait7", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "Processes" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 8')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait8", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "Processes" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 9')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait9", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "Processes" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/GetSetMetadata')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" }, "annotations": [ "ADF.procfwk" ], "type": "AzureSqlTable", "schema": [], "typeProperties": {} }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/Keys')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection to Key Vault for all other ADF linked service credentials required to run the processing framework.", "annotations": [ "ADF.procfwk" ], "type": "AzureKeyVault", "typeProperties": { "baseUrl": "[parameters('Keys_properties_typeProperties_baseUrl')]" } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/PipelineExecutor')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection to the Function App from ADF for calling the processing pipeline function within the orchestration framework.", "annotations": [ "ADF.procfwk" ], "type": "AzureFunction", "typeProperties": { "functionAppUrl": "https://pipelineexecutor.azurewebsites.net", "functionKey": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "ExecutorFunctionAppKey" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/SupportDatabase')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection between ADF and processing framework metadata SQLDB.", "annotations": [ "ADF.procfwk" ], "type": "AzureSqlDatabase", "typeProperties": { "connectionString": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "[parameters('SupportDatabase_properties_typeProperties_connectionString_secretName')]" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/FunctionalTestingTrigger')]", "type": "Microsoft.DataFactory/factories/triggers", "apiVersion": "2018-06-01", "properties": { "description": "Used internally to run the processing framework hourly.", "annotations": [ "ADF.procfwk" ], "runtimeState": "Stopped", "pipelines": [ { "pipelineReference": { "referenceName": "01-Grandparent", "type": "PipelineReference" }, "parameters": {} } ], "type": "ScheduleTrigger", "typeProperties": { "recurrence": { "frequency": "Hour", "interval": 1, "startTime": "2020-04-06T15:00:00.000Z", "timeZone": "UTC" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/pipelines/01-Grandparent')]" ] } ] } ================================================ FILE: ARM Templates/Data Factory/v1.4 Export.json ================================================ { "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { "factoryName": { "type": "string", "metadata": "Data Factory name", "defaultValue": "" }, "Keys_properties_typeProperties_baseUrl": { "type": "string", "defaultValue": "" }, "SupportDatabase_properties_typeProperties_connectionString_secretName": { "type": "string", "defaultValue": "" } }, "variables": { "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" }, "resources": [ { "name": "[concat(parameters('factoryName'), '/01-Grandparent')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk grandparent pipeline used optionally to bootstrap any wider processes in your Data Factory that then calls the processing framework.", "activities": [ { "name": "Framework Processing", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Set Random Waits", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "02-Parent", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": {} } }, { "name": "Set Random Waits", "description": "For functional testing only.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[dbo].[SetRandomWaitValues]" }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ], "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/pipelines/02-Parent')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/02-Parent')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk parent pipeline used to bootstrap the orchestration framework in perform the first level ForEach calls in sequence for the metadata stages.", "activities": [ { "name": "Get Stages", "description": "Returns a distinct list of execution stages within the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Execution Wrapper", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetStages]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Stages", "description": "Top level ForEach to sequentially call all processing stages within the framework metadata. Items for iteration passed from the Get Stages lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Stages", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Tenant Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Subscription Id", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Stages').output.value", "type": "Expression" }, "isSequential": true, "activities": [ { "name": "Stage Executor", "description": "Call to the framework generic child pipeline for a given execution stage.", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Log Stage Preparing", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "03-Child", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "StageId": { "value": "@item().StageId", "type": "Expression" }, "ExecutionId": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" }, "TenantId": { "value": "@activity('Get Tenant Id').output.firstRow.PropertyValue", "type": "Expression" }, "SubscriptionId": { "value": "@activity('Get Subscription Id').output.firstRow.PropertyValue", "type": "Expression" } } } }, { "name": "Log Stage Preparing", "description": "Update the current execution table flagging all pipelines within the stage as preparing.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Check for Blockers", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogStagePreparing]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check for Blockers", "description": "Used to double check and stop the next execution stage if failures and blockers have be incurred. Without this step processing would continue regardless of upstream failures.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[CheckForBlockedPipelines]", "storedProcedureParameters": { "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Execution Wrapper", "description": "Wrapper to reset and restart processing or create a completely new execution instance of the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Metadata Integrity Checks", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[ExecutionWrapper]", "storedProcedureParameters": { "CallingDataFactory": { "type": "String", "value": { "value": "@pipeline().DataFactory", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Archive Execution Log", "description": "After a successful execution run the current execution metadata is moved to the long term logging table by this stored procedure call.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Stages", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[UpdateExecutionLog]" }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Tenant Id", "description": "Returning the Azure Tenant Id from the metadata properties table.", "type": "Lookup", "dependsOn": [ { "activity": "Metadata Integrity Checks", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "TenantId" } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Get Subscription Id", "description": "Returning the Azure Subscription Id from the metadata properties table.", "type": "Lookup", "dependsOn": [ { "activity": "Metadata Integrity Checks", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "SubscriptionId" } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Metadata Integrity Checks", "description": "Performs a series of checks on all metadata held in the framework SQLDB. This is intended to raise errors before an execution run even starts.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[CheckMetadataIntegrity]", "storedProcedureParameters": { "DebugMode": { "value": "false", "type": "Boolean" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ], "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", "[concat(variables('factoryId'), '/pipelines/03-Child')]" ] }, { "name": "[concat(parameters('factoryName'), '/03-Child')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk child pipeline used to execute Worker pipelines within a given execution stage. This pipeline will be called once for each stage, then execute all Workers in parallel.", "activities": [ { "name": "Get Pipelines", "description": "Returns all pipelines from the metadata to be executed within a given processing stage.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelinesInStage]", "storedProcedureParameters": { "StageId": { "type": "Int32", "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Pipelines", "description": "Second level ForEach to run in parallel all pipelines within the stage. Items for iteration passed from the Get Pipelines lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Pipelines", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Pipelines').output.value", "type": "Expression" }, "isSequential": false, "activities": [ { "name": "Execute Pipeline", "description": "The lowest level executor with the metadata framework to call existing processing pipelines within Data Factory. The function called will block processing and wait for an outcome.", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Log Pipeline Running", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Pipeline Params", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get SPN Details", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "ExecutePipelineV2", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n\t\"tenantId\": \"',pipeline().parameters.TenantId,'\",\n\t\"applicationId\": \"',activity('Get SPN Details').output.firstRow.Id,'\",\n\t\"authenticationKey\": \"',activity('Get SPN Details').output.firstRow.Secret,'\",\n\t\"subscriptionId\": \"',pipeline().parameters.SubscriptionId,'\",\n\t\"resourceGroup\": \"',item().ResourceGroupName,'\",\n\t\"factoryName\": \"',item().DataFactoryName,'\",\n\t\"pipelineName\": \"',item().PipelineName,'\"',activity('Get Pipeline Params').output.firstRow.Params,'\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "PipelineExecutor", "type": "LinkedServiceReference" } }, { "name": "Get Pipeline Params", "description": "Returns any parameters from metadata required for the processing pipeline being called. The output can be an empty string if no parameters are required.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelineParameters]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@item().PipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Pipeline Running", "description": "Sets the current pipeline with a status of running within the current execution database table.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get SPN Details", "description": "Return the SPN ID and Secret for the processing pipeline being executed. Called at this level as each pipeline can have a different SPN.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetServicePrincipal]", "storedProcedureParameters": { "DataFactory": { "type": "String", "value": { "value": "@item().DataFactoryName", "type": "Expression" } }, "PipelineName": { "type": "String", "value": { "value": "@item().PipelineName", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Activity Failure", "description": "Handle true failures from calling out to the Azure Function and update the current execution table accordingly so a restart can occur.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@pipeline().RunId", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Running Pipeline Handler", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "04-Infant", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "tenantId": { "value": "@pipeline().parameters.TenantId", "type": "Expression" }, "applicationId": { "value": "@activity('Get SPN Details').output.firstRow.Id", "type": "Expression" }, "authenticationKey": { "value": "@activity('Get SPN Details').output.firstRow.Secret", "type": "Expression" }, "subscriptionId": { "value": "@pipeline().parameters.SubscriptionId", "type": "Expression" }, "resourceGroup": { "value": "@item().ResourceGroupName", "type": "Expression" }, "factoryName": { "value": "@item().DataFactoryName", "type": "Expression" }, "pipelineName": { "value": "@item().PipelineName", "type": "Expression" }, "runId": { "value": "@activity('Execute Pipeline').output.RunId", "type": "Expression" }, "executionId": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "stageId": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "pipelineId": { "value": "@item().PipelineId", "type": "Expression" } } } } ] } } ], "parameters": { "StageId": { "type": "int" }, "ExecutionId": { "type": "string" }, "TenantId": { "type": "string" }, "SubscriptionId": { "type": "string" } }, "variables": { "FunctionBody": { "type": "String" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/PipelineExecutor')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", "[concat(variables('factoryId'), '/pipelines/04-Infant')]" ] }, { "name": "[concat(parameters('factoryName'), '/04-Infant')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk infant pipeline used to check when the processing pipeline called by the Child completes and passes the resulting status back to the metadata database.", "activities": [ { "name": "Wait Until Pipeline Completes", "description": "Loops until the pipeline called completes.\n\nSimple status:\n- Running = new iteration.\n- Done = break.", "type": "Until", "dependsOn": [ { "activity": "Get Wait Duration", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@equals('Done',activity('Get Pipeline Status').output.SimpleStatus)", "type": "Expression" }, "activities": [ { "name": "Get Pipeline Status", "description": "Checks the status of a given processing pipeline and provides the value for the downstream framework activities to act upon.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "CheckPipelineStatus", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',pipeline().parameters.TenantId,'\",\n \"applicationId\": \"',pipeline().parameters.applicationId,'\",\n \"authenticationKey\": \"',pipeline().parameters.authenticationKey,'\",\n \"subscriptionId\": \"',pipeline().parameters.subscriptionId,'\",\n \"resourceGroup\": \"',pipeline().parameters.resourceGroup,'\",\n \"factoryName\": \"',pipeline().parameters.factoryName,'\",\n \"pipelineName\": \"',pipeline().parameters.pipelineName,'\",\n \"runId\": \"',pipeline().parameters.runId,'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "PipelineExecutor", "type": "LinkedServiceReference" } }, { "name": "Wait If Running", "description": "True = Do nothing.\nFalse = Wait.", "type": "IfCondition", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@equals('Done',activity('Get Pipeline Status').output.SimpleStatus)", "type": "Expression" }, "ifFalseActivities": [ { "name": "Wait for Pipeline", "description": "The processing pipeline is still running so Wait before checking its status again.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@activity('Get Wait Duration').output.firstRow.PropertyValue", "type": "Expression" } } } ] } }, { "name": "Wait to Retry Function", "type": "Wait", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Failed" ] } ], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@activity('Get Wait Duration').output.firstRow.PropertyValue", "type": "Expression" } } }, { "name": "Set Last Update", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineLastStatusCheck]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ], "timeout": "7.00:00:00" } }, { "name": "Pipeline Result", "description": "Receives the outcome from the function execution for a given processing pipeline and updates the current execution table with different pipelines status values depending on the result (case).", "type": "Switch", "dependsOn": [ { "activity": "Wait Until Pipeline Completes", "dependencyConditions": [ "Completed" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@activity('Get Pipeline Status').output.Status", "type": "Expression" }, "cases": [ { "value": "Succeeded", "activities": [ { "name": "Pipeline Status Succeeded", "description": "Updates the current execution table with a pipeline status of success if the function outcome is succeeded.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Failed", "activities": [ { "name": "Pipeline Status Failed", "description": "Updates the current execution table with a pipeline status of failed if the function outcome is failed. Also blocks pipelines in the downstream execution stage.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@activity('Get Pipeline Status').output.RunId", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } ], "defaultActivities": [ { "name": "Pipeline Status Unknown", "description": "Updates the current execution table with a pipeline status of unknown if the function returns an unexpected outcome.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineUnknown]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Get Wait Duration", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "PipelineStatusCheckDuration" } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } } ], "parameters": { "tenantId": { "type": "string" }, "applicationId": { "type": "string" }, "authenticationKey": { "type": "string" }, "subscriptionId": { "type": "string" }, "resourceGroup": { "type": "string" }, "factoryName": { "type": "string" }, "pipelineName": { "type": "string" }, "runId": { "type": "string" }, "executionId": { "type": "string" }, "stageId": { "type": "int" }, "pipelineId": { "type": "int" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/PipelineExecutor')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/Intention Error')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Call Fail Procedure", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[dbo].[FailProcedure]" }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ], "folder": { "name": "Processes" }, "annotations": [] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/Wait 1')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "Processes" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 10')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait10", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "Processes" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 2')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait2", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "Processes" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 3')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait3", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "Processes" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 4')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait4", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "Processes" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 5')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait5", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "Processes" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 6')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait6", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "Processes" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 7')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait7", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "Processes" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 8')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait8", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "Processes" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 9')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait9", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "Processes" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/GetSetMetadata')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" }, "annotations": [ "ADF.procfwk" ], "type": "AzureSqlTable", "schema": [], "typeProperties": {} }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/Keys')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection to Key Vault for all other ADF linked service credentials required to run the processing framework.", "annotations": [ "ADF.procfwk" ], "type": "AzureKeyVault", "typeProperties": { "baseUrl": "[parameters('Keys_properties_typeProperties_baseUrl')]" } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/PipelineExecutor')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection to the Function App from ADF for calling the processing pipeline function within the orchestration framework.", "annotations": [ "ADF.procfwk" ], "type": "AzureFunction", "typeProperties": { "functionAppUrl": "https://pipelineexecutor.azurewebsites.net", "functionKey": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "ExecutorFunctionAppKey" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/SupportDatabase')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection between ADF and processing framework metadata SQLDB.", "annotations": [ "ADF.procfwk" ], "type": "AzureSqlDatabase", "typeProperties": { "connectionString": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "[parameters('SupportDatabase_properties_typeProperties_connectionString_secretName')]" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/FunctionalTestingTrigger')]", "type": "Microsoft.DataFactory/factories/triggers", "apiVersion": "2018-06-01", "properties": { "description": "Used for functional testing of the framework in a dedicated environment.", "annotations": [ "ADF.procfwk" ], "runtimeState": "Stopped", "pipelines": [ { "pipelineReference": { "referenceName": "01-Grandparent", "type": "PipelineReference" }, "parameters": {} } ], "type": "ScheduleTrigger", "typeProperties": { "recurrence": { "frequency": "Hour", "interval": 1, "startTime": "2020-04-06T15:00:00.000Z", "timeZone": "UTC" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/pipelines/01-Grandparent')]" ] } ] } ================================================ FILE: ARM Templates/Data Factory/v1.5 Export.json ================================================ { "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { "factoryName": { "type": "string", "metadata": "Data Factory name", "defaultValue": "" }, "Keys_properties_typeProperties_baseUrl": { "type": "string", "defaultValue": "" }, "SupportDatabase_properties_typeProperties_connectionString_secretName": { "type": "string", "defaultValue": "" } }, "variables": { "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" }, "resources": [ { "name": "[concat(parameters('factoryName'), '/01-Grandparent')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk grandparent pipeline used optionally to bootstrap any wider processes in your Data Factory that then calls the processing framework.", "activities": [ { "name": "Framework Processing", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Set Random Waits", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "02-Parent", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": {} } }, { "name": "Set Random Waits", "description": "For functional testing only.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[dbo].[SetRandomWaitValues]" }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ], "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/pipelines/02-Parent')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/02-Parent')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk parent pipeline used to bootstrap the orchestration framework in perform the first level ForEach calls in sequence for the metadata stages.", "activities": [ { "name": "Get Stages", "description": "Returns a distinct list of execution stages within the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Execution Wrapper", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetStages]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Stages", "description": "Top level ForEach to sequentially call all processing stages within the framework metadata. Items for iteration passed from the Get Stages lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Stages", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Tenant Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Subscription Id", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Stages').output.value", "type": "Expression" }, "isSequential": true, "activities": [ { "name": "Stage Executor", "description": "Call to the framework generic child pipeline for a given execution stage.", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Log Stage Preparing", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "03-Child", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "StageId": { "value": "@item().StageId", "type": "Expression" }, "ExecutionId": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" }, "TenantId": { "value": "@activity('Get Tenant Id').output.firstRow.PropertyValue", "type": "Expression" }, "SubscriptionId": { "value": "@activity('Get Subscription Id').output.firstRow.PropertyValue", "type": "Expression" } } } }, { "name": "Log Stage Preparing", "description": "Update the current execution table flagging all pipelines within the stage as preparing.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Check for Blockers", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogStagePreparing]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check for Blockers", "description": "Used to double check and stop the next execution stage if failures and blockers have be incurred. Without this step processing would continue regardless of upstream failures.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[CheckForBlockedPipelines]", "storedProcedureParameters": { "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Execution Wrapper", "description": "Wrapper to reset and restart processing or create a completely new execution instance of the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Metadata Integrity Checks", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[ExecutionWrapper]", "storedProcedureParameters": { "CallingDataFactory": { "type": "String", "value": { "value": "@pipeline().DataFactory", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Archive Execution Log", "description": "After a successful execution run the current execution metadata is moved to the long term logging table by this stored procedure call.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Stages", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[UpdateExecutionLog]" }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Tenant Id", "description": "Returning the Azure Tenant Id from the metadata properties table.", "type": "Lookup", "dependsOn": [ { "activity": "Metadata Integrity Checks", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "TenantId" } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Get Subscription Id", "description": "Returning the Azure Subscription Id from the metadata properties table.", "type": "Lookup", "dependsOn": [ { "activity": "Metadata Integrity Checks", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "SubscriptionId" } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Metadata Integrity Checks", "description": "Performs a series of checks on all metadata held in the framework SQLDB. This is intended to raise errors before an execution run even starts.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[CheckMetadataIntegrity]", "storedProcedureParameters": { "DebugMode": { "value": "false", "type": "Boolean" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ], "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", "[concat(variables('factoryId'), '/pipelines/03-Child')]" ] }, { "name": "[concat(parameters('factoryName'), '/03-Child')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk child pipeline used to execute Worker pipelines within a given execution stage. This pipeline will be called once for each stage, then execute all Workers in parallel.", "activities": [ { "name": "Get Pipelines", "description": "Returns all pipelines from the metadata to be executed within a given processing stage.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelinesInStage]", "storedProcedureParameters": { "StageId": { "type": "Int32", "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Pipelines", "description": "Second level ForEach to run in parallel all pipelines within the stage. Items for iteration passed from the Get Pipelines lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Pipelines", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Pipelines').output.value", "type": "Expression" }, "isSequential": false, "activities": [ { "name": "Execute Pipeline", "description": "The lowest level executor with the metadata framework to call existing processing pipelines within Data Factory. The function called will block processing and wait for an outcome.", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Log Pipeline Running", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Pipeline Params", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get SPN Details", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "ExecutePipelineV2", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n\t\"tenantId\": \"',pipeline().parameters.TenantId,'\",\n\t\"applicationId\": \"',activity('Get SPN Details').output.firstRow.Id,'\",\n\t\"authenticationKey\": \"',activity('Get SPN Details').output.firstRow.Secret,'\",\n\t\"subscriptionId\": \"',pipeline().parameters.SubscriptionId,'\",\n\t\"resourceGroup\": \"',item().ResourceGroupName,'\",\n\t\"factoryName\": \"',item().DataFactoryName,'\",\n\t\"pipelineName\": \"',item().PipelineName,'\"',activity('Get Pipeline Params').output.firstRow.Params,'\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "PipelineExecutor", "type": "LinkedServiceReference" } }, { "name": "Get Pipeline Params", "description": "Returns any parameters from metadata required for the processing pipeline being called. The output can be an empty string if no parameters are required.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelineParameters]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@item().PipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Pipeline Running", "description": "Sets the current pipeline with a status of running within the current execution database table.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get SPN Details", "description": "Return the SPN ID and Secret for the processing pipeline being executed. Called at this level as each pipeline can have a different SPN.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetServicePrincipal]", "storedProcedureParameters": { "DataFactory": { "type": "String", "value": { "value": "@item().DataFactoryName", "type": "Expression" } }, "PipelineName": { "type": "String", "value": { "value": "@item().PipelineName", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Activity Failure", "description": "Handle true failures from calling out to the Azure Function and update the current execution table accordingly so a restart can occur.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@pipeline().RunId", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Running Pipeline Handler", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "04-Infant", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "tenantId": { "value": "@pipeline().parameters.TenantId", "type": "Expression" }, "applicationId": { "value": "@activity('Get SPN Details').output.firstRow.Id", "type": "Expression" }, "authenticationKey": { "value": "@activity('Get SPN Details').output.firstRow.Secret", "type": "Expression" }, "subscriptionId": { "value": "@pipeline().parameters.SubscriptionId", "type": "Expression" }, "resourceGroup": { "value": "@item().ResourceGroupName", "type": "Expression" }, "factoryName": { "value": "@item().DataFactoryName", "type": "Expression" }, "pipelineName": { "value": "@item().PipelineName", "type": "Expression" }, "runId": { "value": "@activity('Execute Pipeline').output.RunId", "type": "Expression" }, "executionId": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "stageId": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "pipelineId": { "value": "@item().PipelineId", "type": "Expression" } } } }, { "name": "Set Run Id", "description": "Provide the actual ADF run ID back to the current execution table for long term logging and alignment between the metadata other Azure monitoring tools.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunId]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@activity('Execute Pipeline').output.RunId", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } } ], "parameters": { "StageId": { "type": "int" }, "ExecutionId": { "type": "string" }, "TenantId": { "type": "string" }, "SubscriptionId": { "type": "string" } }, "variables": { "FunctionBody": { "type": "String" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/PipelineExecutor')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", "[concat(variables('factoryId'), '/pipelines/04-Infant')]" ] }, { "name": "[concat(parameters('factoryName'), '/04-Infant')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk infant pipeline used to check when the processing pipeline called by the Child completes and passes the resulting status back to the metadata database.", "activities": [ { "name": "Wait Until Pipeline Completes", "description": "Loops until the pipeline called completes.\n\nSimple status:\n- Running = new iteration.\n- Done = break.", "type": "Until", "dependsOn": [ { "activity": "Get Wait Duration", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@equals('Done',activity('Get Pipeline Status').output.SimpleStatus)", "type": "Expression" }, "activities": [ { "name": "Get Pipeline Status", "description": "Checks the status of a given processing pipeline and provides the value for the downstream framework activities to act upon.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "CheckPipelineStatus", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',pipeline().parameters.TenantId,'\",\n \"applicationId\": \"',pipeline().parameters.applicationId,'\",\n \"authenticationKey\": \"',pipeline().parameters.authenticationKey,'\",\n \"subscriptionId\": \"',pipeline().parameters.subscriptionId,'\",\n \"resourceGroup\": \"',pipeline().parameters.resourceGroup,'\",\n \"factoryName\": \"',pipeline().parameters.factoryName,'\",\n \"pipelineName\": \"',pipeline().parameters.pipelineName,'\",\n \"runId\": \"',pipeline().parameters.runId,'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "PipelineExecutor", "type": "LinkedServiceReference" } }, { "name": "Wait If Running", "description": "True = Do nothing.\nFalse = Wait.", "type": "IfCondition", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@equals('Done',activity('Get Pipeline Status').output.SimpleStatus)", "type": "Expression" }, "ifFalseActivities": [ { "name": "Wait for Pipeline", "description": "The processing pipeline is still running so Wait before checking its status again.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@activity('Get Wait Duration').output.firstRow.PropertyValue", "type": "Expression" } } } ] } }, { "name": "Wait to Retry Function", "type": "Wait", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Failed" ] } ], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@activity('Get Wait Duration').output.firstRow.PropertyValue", "type": "Expression" } } }, { "name": "Set Last Update", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineLastStatusCheck]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ], "timeout": "7.00:00:00" } }, { "name": "Pipeline Result", "description": "Receives the outcome from the function execution for a given processing pipeline and updates the current execution table with different pipelines status values depending on the result (case).", "type": "Switch", "dependsOn": [ { "activity": "Wait Until Pipeline Completes", "dependencyConditions": [ "Completed" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@activity('Get Pipeline Status').output.Status", "type": "Expression" }, "cases": [ { "value": "Succeeded", "activities": [ { "name": "Pipeline Status Succeeded", "description": "Updates the current execution table with a pipeline status of success if the function outcome is succeeded.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Failed", "activities": [ { "name": "Pipeline Status Failed", "description": "Updates the current execution table with a pipeline status of failed if the function outcome is failed. Also blocks pipelines in the downstream execution stage.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@activity('Get Pipeline Status').output.RunId", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } ], "defaultActivities": [ { "name": "Pipeline Status Unknown", "description": "Updates the current execution table with a pipeline status of unknown if the function returns an unexpected outcome.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineUnknown]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Get Wait Duration", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "PipelineStatusCheckDuration" } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } } ], "parameters": { "tenantId": { "type": "string" }, "applicationId": { "type": "string" }, "authenticationKey": { "type": "string" }, "subscriptionId": { "type": "string" }, "resourceGroup": { "type": "string" }, "factoryName": { "type": "string" }, "pipelineName": { "type": "string" }, "runId": { "type": "string" }, "executionId": { "type": "string" }, "stageId": { "type": "int" }, "pipelineId": { "type": "int" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/PipelineExecutor')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/Intentional Error')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Call Fail Procedure", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Wait1", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[dbo].[FailProcedure]", "storedProcedureParameters": { "RaiseError": { "value": { "value": "@pipeline().parameters.RaiseErrors", "type": "Expression" }, "type": "String" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Wait1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 10 } }, { "name": "Call Fail Notebook", "type": "DatabricksNotebook", "dependsOn": [ { "activity": "Wait1", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "notebookPath": "/Playground/Throw Exception", "baseParameters": { "RaiseError": { "value": "@pipeline().parameters.RaiseErrors", "type": "Expression" } } }, "linkedServiceName": { "referenceName": "BricksOfData", "type": "LinkedServiceReference" } } ], "parameters": { "RaiseErrors": { "type": "string", "defaultValue": "false" } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", "[concat(variables('factoryId'), '/linkedServices/BricksOfData')]" ] }, { "name": "[concat(parameters('factoryName'), '/Wait 1')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 10')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait10", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 2')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait2", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 3')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait3", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 4')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait4", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 5')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait5", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 6')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait6", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 7')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait7", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 8')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait8", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 9')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait9", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/GetSetMetadata')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" }, "annotations": [ "ADF.procfwk" ], "type": "AzureSqlTable", "schema": [], "typeProperties": {} }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/BricksOfData')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [], "type": "AzureDatabricks", "typeProperties": { "domain": "https://northeurope.azuredatabricks.net", "accessToken": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "BricksOfDataToken", "secretVersion": "" }, "existingClusterId": "0422-090117-slots899" } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/Keys')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection to Key Vault for all other ADF linked service credentials required to run the processing framework.", "annotations": [ "ADF.procfwk" ], "type": "AzureKeyVault", "typeProperties": { "baseUrl": "[parameters('Keys_properties_typeProperties_baseUrl')]" } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/PipelineExecutor')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection to the Function App from ADF for calling the processing pipeline function within the orchestration framework.", "annotations": [ "ADF.procfwk" ], "type": "AzureFunction", "typeProperties": { "functionAppUrl": "https://pipelineexecutor.azurewebsites.net", "functionKey": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "ExecutorFunctionAppKey" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/SupportDatabase')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection between ADF and processing framework metadata SQLDB.", "annotations": [ "ADF.procfwk" ], "type": "AzureSqlDatabase", "typeProperties": { "connectionString": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "[parameters('SupportDatabase_properties_typeProperties_connectionString_secretName')]" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/FunctionalTestingTrigger')]", "type": "Microsoft.DataFactory/factories/triggers", "apiVersion": "2018-06-01", "properties": { "description": "Used for functional testing of the framework in a dedicated environment.", "annotations": [ "ADF.procfwk" ], "runtimeState": "Stopped", "pipelines": [ { "pipelineReference": { "referenceName": "01-Grandparent", "type": "PipelineReference" }, "parameters": {} } ], "type": "ScheduleTrigger", "typeProperties": { "recurrence": { "frequency": "Hour", "interval": 2, "startTime": "2020-04-06T15:00:00.000Z", "timeZone": "UTC" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/pipelines/01-Grandparent')]" ] } ] } ================================================ FILE: ARM Templates/Data Factory/v1.6 Export.json ================================================ { "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { "factoryName": { "type": "string", "metadata": "Data Factory name", "defaultValue": "" }, "Keys_properties_typeProperties_baseUrl": { "type": "string", "defaultValue": "" }, "SupportDatabase_properties_typeProperties_connectionString_secretName": { "type": "string", "defaultValue": "" } }, "variables": { "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" }, "resources": [ { "name": "[concat(parameters('factoryName'), '/01-Grandparent')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk grandparent pipeline used optionally to bootstrap any wider processes in your Data Factory that then calls the processing framework.", "activities": [ { "name": "Framework Processing", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Set Random Waits", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "02-Parent", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": {} } }, { "name": "Set Random Waits", "description": "For functional testing only.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[dbo].[SetRandomWaitValues]" }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ], "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/pipelines/02-Parent')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/02-Parent')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk parent pipeline used to bootstrap the orchestration framework in perform the first level ForEach calls in sequence for the metadata stages.", "activities": [ { "name": "Get Stages", "description": "Returns a distinct list of execution stages within the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Execution Wrapper", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetStages]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Stages", "description": "Top level ForEach to sequentially call all processing stages within the framework metadata. Items for iteration passed from the Get Stages lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Stages", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Tenant Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Subscription Id", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Stages').output.value", "type": "Expression" }, "isSequential": true, "activities": [ { "name": "Stage Executor", "description": "Call to the framework generic child pipeline for a given execution stage.", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Log Stage Preparing", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "03-Child", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "StageId": { "value": "@item().StageId", "type": "Expression" }, "ExecutionId": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" }, "TenantId": { "value": "@activity('Get Tenant Id').output.firstRow.PropertyValue", "type": "Expression" }, "SubscriptionId": { "value": "@activity('Get Subscription Id').output.firstRow.PropertyValue", "type": "Expression" } } } }, { "name": "Log Stage Preparing", "description": "Update the current execution table flagging all pipelines within the stage as preparing.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Check for Blockers", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogStagePreparing]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check for Blockers", "description": "Used to double check and stop the next execution stage if failures and blockers have be incurred. Without this step processing would continue regardless of upstream failures.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[CheckForBlockedPipelines]", "storedProcedureParameters": { "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Execution Wrapper", "description": "Wrapper to reset and restart processing or create a completely new execution instance of the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Metadata Integrity Checks", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[ExecutionWrapper]", "storedProcedureParameters": { "CallingDataFactory": { "type": "String", "value": { "value": "@pipeline().DataFactory", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Archive Execution Log", "description": "After a successful execution run the current execution metadata is moved to the long term logging table by this stored procedure call.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Stages", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[UpdateExecutionLog]" }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Tenant Id", "description": "Returning the Azure Tenant Id from the metadata properties table.", "type": "Lookup", "dependsOn": [ { "activity": "Metadata Integrity Checks", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "TenantId" } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Get Subscription Id", "description": "Returning the Azure Subscription Id from the metadata properties table.", "type": "Lookup", "dependsOn": [ { "activity": "Metadata Integrity Checks", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "SubscriptionId" } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Metadata Integrity Checks", "description": "Performs a series of checks on all metadata held in the framework SQLDB. This is intended to raise errors before an execution run even starts.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[CheckMetadataIntegrity]", "storedProcedureParameters": { "DebugMode": { "value": "false", "type": "Boolean" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ], "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", "[concat(variables('factoryId'), '/pipelines/03-Child')]" ] }, { "name": "[concat(parameters('factoryName'), '/03-Child')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk child pipeline used to execute Worker pipelines within a given execution stage. This pipeline will be called once for each stage, then execute all Workers in parallel.", "activities": [ { "name": "Get Pipelines", "description": "Returns all pipelines from the metadata to be executed within a given processing stage.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelinesInStage]", "storedProcedureParameters": { "StageId": { "type": "Int32", "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Pipelines", "description": "Second level ForEach to run in parallel all pipelines within the stage. Items for iteration passed from the Get Pipelines lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Pipelines", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Pipelines').output.value", "type": "Expression" }, "isSequential": false, "activities": [ { "name": "Execute Pipeline", "description": "The lowest level executor with the metadata framework to call existing processing pipelines within Data Factory. The function called will block processing and wait for an outcome.", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Log Pipeline Running", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Pipeline Params", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get SPN Details", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "ExecutePipeline", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n\t\"tenantId\": \"',pipeline().parameters.TenantId,'\",\n\t\"applicationId\": \"',activity('Get SPN Details').output.firstRow.Id,'\",\n\t\"authenticationKey\": \"',activity('Get SPN Details').output.firstRow.Secret,'\",\n\t\"subscriptionId\": \"',pipeline().parameters.SubscriptionId,'\",\n\t\"resourceGroup\": \"',item().ResourceGroupName,'\",\n\t\"factoryName\": \"',item().DataFactoryName,'\",\n\t\"pipelineName\": \"',item().PipelineName,'\"',activity('Get Pipeline Params').output.firstRow.Params,'\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Get Pipeline Params", "description": "Returns any parameters from metadata required for the processing pipeline being called. The output can be an empty string if no parameters are required.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelineParameters]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@item().PipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Pipeline Running", "description": "Sets the current pipeline with a status of running within the current execution database table.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get SPN Details", "description": "Return the SPN ID and Secret for the processing pipeline being executed. Called at this level as each pipeline can have a different SPN.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetServicePrincipal]", "storedProcedureParameters": { "DataFactory": { "type": "String", "value": { "value": "@item().DataFactoryName", "type": "Expression" } }, "PipelineName": { "type": "String", "value": { "value": "@item().PipelineName", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Activity Failure", "description": "Handle true failures from calling out to the Azure Function and update the current execution table accordingly so a restart can occur.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Running Pipeline Handler", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "04-Infant", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "tenantId": { "value": "@pipeline().parameters.TenantId", "type": "Expression" }, "applicationId": { "value": "@activity('Get SPN Details').output.firstRow.Id", "type": "Expression" }, "authenticationKey": { "value": "@activity('Get SPN Details').output.firstRow.Secret", "type": "Expression" }, "subscriptionId": { "value": "@pipeline().parameters.SubscriptionId", "type": "Expression" }, "resourceGroup": { "value": "@item().ResourceGroupName", "type": "Expression" }, "factoryName": { "value": "@item().DataFactoryName", "type": "Expression" }, "pipelineName": { "value": "@item().PipelineName", "type": "Expression" }, "runId": { "value": "@activity('Execute Pipeline').output.RunId", "type": "Expression" }, "executionId": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "stageId": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "pipelineId": { "value": "@item().PipelineId", "type": "Expression" } } } }, { "name": "Set Run Id", "description": "Provide the actual ADF run ID back to the current execution table for long term logging and alignment between the metadata other Azure monitoring tools.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunId]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@activity('Execute Pipeline').output.RunId", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } } ], "parameters": { "StageId": { "type": "int" }, "ExecutionId": { "type": "string" }, "TenantId": { "type": "string" }, "SubscriptionId": { "type": "string" } }, "variables": { "FunctionBody": { "type": "String" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/FrameworkFunctions')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", "[concat(variables('factoryId'), '/pipelines/04-Infant')]" ] }, { "name": "[concat(parameters('factoryName'), '/04-Infant')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk infant pipeline used to check when the processing pipeline called by the Child completes and passes the resulting status back to the metadata database.", "activities": [ { "name": "Wait Until Pipeline Completes", "description": "Loops until the pipeline called completes.\n\nSimple status:\n- Running = new iteration.\n- Done = break.", "type": "Until", "dependsOn": [ { "activity": "Get Wait Duration", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@equals('Done',activity('Get Pipeline Status').output.SimpleStatus)", "type": "Expression" }, "activities": [ { "name": "Get Pipeline Status", "description": "Checks the status of a given processing pipeline and provides the value for the downstream framework activities to act upon.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "CheckPipelineStatus", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',pipeline().parameters.TenantId,'\",\n \"applicationId\": \"',pipeline().parameters.applicationId,'\",\n \"authenticationKey\": \"',pipeline().parameters.authenticationKey,'\",\n \"subscriptionId\": \"',pipeline().parameters.subscriptionId,'\",\n \"resourceGroup\": \"',pipeline().parameters.resourceGroup,'\",\n \"factoryName\": \"',pipeline().parameters.factoryName,'\",\n \"pipelineName\": \"',pipeline().parameters.pipelineName,'\",\n \"runId\": \"',pipeline().parameters.runId,'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Wait If Running", "description": "True = Do nothing.\nFalse = Wait.", "type": "IfCondition", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@equals('Done',activity('Get Pipeline Status').output.SimpleStatus)", "type": "Expression" }, "ifFalseActivities": [ { "name": "Wait for Pipeline", "description": "The processing pipeline is still running so Wait before checking its status again.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@activity('Get Wait Duration').output.firstRow.PropertyValue", "type": "Expression" } } } ] } }, { "name": "Wait to Retry Function", "type": "Wait", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Failed" ] } ], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@activity('Get Wait Duration').output.firstRow.PropertyValue", "type": "Expression" } } }, { "name": "Set Last Update", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineLastStatusCheck]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ], "timeout": "7.00:00:00" } }, { "name": "Pipeline Result", "description": "Receives the outcome from the function execution for a given processing pipeline and updates the current execution table with different pipelines status values depending on the result (case).", "type": "Switch", "dependsOn": [ { "activity": "Wait Until Pipeline Completes", "dependencyConditions": [ "Completed" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@activity('Get Pipeline Status').output.Status", "type": "Expression" }, "cases": [ { "value": "Succeeded", "activities": [ { "name": "Pipeline Status Succeeded", "description": "Updates the current execution table with a pipeline status of success if the function outcome is succeeded.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Failed", "activities": [ { "name": "Pipeline Status Failed", "description": "Updates the current execution table with a pipeline status of failed if the function outcome is failed. Also blocks pipelines in the downstream execution stage.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@activity('Get Pipeline Status').output.RunId", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Error Details", "description": "Get the activity error details for the run ID of the worker pipeline called. Returns an array of all errors.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "GetActivityErrors", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',pipeline().parameters.TenantId,'\",\n \"applicationId\": \"',pipeline().parameters.applicationId,'\",\n \"authenticationKey\": \"',pipeline().parameters.authenticationKey,'\",\n \"subscriptionId\": \"',pipeline().parameters.subscriptionId,'\",\n \"resourceGroup\": \"',pipeline().parameters.resourceGroup,'\",\n \"factoryName\": \"',pipeline().parameters.factoryName,'\",\n \"pipelineName\": \"',pipeline().parameters.pipelineName,'\",\n \"runId\": \"',pipeline().parameters.runId,'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Log Error Details", "description": "Parses pipeline error details and persists them to the metadata database error log table.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Error Details", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetErrorLogDetails]", "storedProcedureParameters": { "JsonErrorDetails": { "value": { "value": "@string(activity('Get Error Details').output)", "type": "Expression" }, "type": "String" }, "LocalExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } ], "defaultActivities": [ { "name": "Pipeline Status Unknown", "description": "Updates the current execution table with a pipeline status of unknown if the function returns an unexpected outcome.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineUnknown]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Get Wait Duration", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "PipelineStatusCheckDuration" } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } } ], "parameters": { "tenantId": { "type": "string" }, "applicationId": { "type": "string" }, "authenticationKey": { "type": "string" }, "subscriptionId": { "type": "string" }, "resourceGroup": { "type": "string" }, "factoryName": { "type": "string" }, "pipelineName": { "type": "string" }, "runId": { "type": "string" }, "executionId": { "type": "string" }, "stageId": { "type": "int" }, "pipelineId": { "type": "int" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/FrameworkFunctions')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/Intentional Error')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Call Fail Procedure", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Wait1", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[dbo].[FailProcedure]", "storedProcedureParameters": { "RaiseError": { "value": { "value": "@pipeline().parameters.RaiseErrors", "type": "Expression" }, "type": "String" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Wait1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } }, { "name": "Call Fail Notebook", "type": "DatabricksNotebook", "dependsOn": [ { "activity": "Wait1", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "notebookPath": "/Playground/Throw Exception", "baseParameters": { "RaiseError": { "value": "@pipeline().parameters.RaiseErrors", "type": "Expression" } } }, "linkedServiceName": { "referenceName": "BricksOfData", "type": "LinkedServiceReference" } } ], "parameters": { "RaiseErrors": { "type": "string", "defaultValue": "false" }, "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", "[concat(variables('factoryId'), '/linkedServices/BricksOfData')]" ] }, { "name": "[concat(parameters('factoryName'), '/Wait 1')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 10')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait10", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 2')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait2", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 3')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait3", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 4')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait4", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 5')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait5", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 6')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait6", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 7')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait7", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 8')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait8", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 9')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait9", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 90 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/GetSetMetadata')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" }, "annotations": [ "ADF.procfwk" ], "type": "AzureSqlTable", "schema": [], "typeProperties": {} }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/BricksOfData')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [], "type": "AzureDatabricks", "typeProperties": { "domain": "https://northeurope.azuredatabricks.net", "accessToken": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "BricksOfDataToken", "secretVersion": "" }, "existingClusterId": "0422-090117-slots899" } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/FrameworkFunctions')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [ "ADF.procfwk" ], "type": "AzureFunction", "typeProperties": { "functionAppUrl": "https://frameworksupportfunctions.azurewebsites.net", "functionKey": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "FrameworkFunctionsKey" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/Keys')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection to Key Vault for all other ADF linked service credentials required to run the processing framework.", "annotations": [ "ADF.procfwk" ], "type": "AzureKeyVault", "typeProperties": { "baseUrl": "[parameters('Keys_properties_typeProperties_baseUrl')]" } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/SupportDatabase')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection between ADF and processing framework metadata SQLDB.", "annotations": [ "ADF.procfwk" ], "type": "AzureSqlDatabase", "typeProperties": { "connectionString": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "[parameters('SupportDatabase_properties_typeProperties_connectionString_secretName')]" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/FunctionalTestingTrigger')]", "type": "Microsoft.DataFactory/factories/triggers", "apiVersion": "2018-06-01", "properties": { "description": "Used for functional testing of the framework in a dedicated environment.", "annotations": [ "ADF.procfwk" ], "runtimeState": "Stopped", "pipelines": [ { "pipelineReference": { "referenceName": "01-Grandparent", "type": "PipelineReference" }, "parameters": {} } ], "type": "ScheduleTrigger", "typeProperties": { "recurrence": { "frequency": "Hour", "interval": 2, "startTime": "2020-04-06T15:00:00.000Z", "timeZone": "UTC" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/pipelines/01-Grandparent')]" ] } ] } ================================================ FILE: ARM Templates/Data Factory/v1.7 Export.json ================================================ { "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { "factoryName": { "type": "string", "metadata": "Data Factory name", "defaultValue": "" }, "Keys_properties_typeProperties_baseUrl": { "type": "string", "defaultValue": "" }, "SupportDatabase_properties_typeProperties_connectionString_secretName": { "type": "string", "defaultValue": "" } }, "variables": { "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" }, "resources": [ { "name": "[concat(parameters('factoryName'), '/01-Grandparent')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk grandparent pipeline used optionally to bootstrap any wider processes in your Data Factory that then calls the processing framework.", "activities": [ { "name": "Framework Processing", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Set Random Waits", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "02-Parent", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": {} } }, { "name": "Set Random Waits", "description": "For functional testing only.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[dbo].[SetRandomWaitValues]" }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ], "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/pipelines/02-Parent')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/02-Parent')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk parent pipeline used to bootstrap the orchestration framework in perform the first level ForEach calls in sequence for the metadata stages.", "activities": [ { "name": "Get Stages", "description": "Returns a distinct list of execution stages within the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Execution Wrapper", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetStages]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Stages", "description": "Top level ForEach to sequentially call all processing stages within the framework metadata. Items for iteration passed from the Get Stages lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Stages", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Tenant Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Subscription Id", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Stages').output.value", "type": "Expression" }, "isSequential": true, "activities": [ { "name": "Stage Executor", "description": "Call to the framework generic child pipeline for a given execution stage.", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Log Stage Preparing", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "03-Child", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "StageId": { "value": "@item().StageId", "type": "Expression" }, "ExecutionId": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" }, "TenantId": { "value": "@activity('Get Tenant Id').output.firstRow.PropertyValue", "type": "Expression" }, "SubscriptionId": { "value": "@activity('Get Subscription Id').output.firstRow.PropertyValue", "type": "Expression" } } } }, { "name": "Log Stage Preparing", "description": "Update the current execution table flagging all pipelines within the stage as preparing.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Check for Blockers", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogStagePreparing]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check for Blockers", "description": "Used to double check and stop the next execution stage if failures and blockers have be incurred. Without this step processing would continue regardless of upstream failures.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[CheckForBlockedPipelines]", "storedProcedureParameters": { "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Execution Wrapper", "description": "Wrapper to reset and restart processing or create a completely new execution instance of the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Metadata Integrity Checks", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[ExecutionWrapper]", "storedProcedureParameters": { "CallingDataFactory": { "type": "String", "value": { "value": "@pipeline().DataFactory", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Archive Execution Log", "description": "After a successful execution run the current execution metadata is moved to the long term logging table by this stored procedure call.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Stages", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[UpdateExecutionLog]" }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Tenant Id", "description": "Returning the Azure Tenant Id from the metadata properties table.", "type": "Lookup", "dependsOn": [ { "activity": "Metadata Integrity Checks", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "TenantId" } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Get Subscription Id", "description": "Returning the Azure Subscription Id from the metadata properties table.", "type": "Lookup", "dependsOn": [ { "activity": "Metadata Integrity Checks", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "SubscriptionId" } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Metadata Integrity Checks", "description": "Performs a series of checks on all metadata held in the framework SQLDB. This is intended to raise errors before an execution run even starts.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[CheckMetadataIntegrity]", "storedProcedureParameters": { "DebugMode": { "value": "false", "type": "Boolean" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ], "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", "[concat(variables('factoryId'), '/pipelines/03-Child')]" ] }, { "name": "[concat(parameters('factoryName'), '/03-Child')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk child pipeline used to execute Worker pipelines within a given execution stage. This pipeline will be called once for each stage, then execute all Workers in parallel.", "activities": [ { "name": "Get Pipelines", "description": "Returns all pipelines from the metadata to be executed within a given processing stage.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelinesInStage]", "storedProcedureParameters": { "StageId": { "type": "Int32", "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Pipelines", "description": "Second level ForEach to run in parallel all pipelines within the stage. Items for iteration passed from the Get Pipelines lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Pipelines", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Pipelines').output.value", "type": "Expression" }, "isSequential": false, "batchCount": 40, "activities": [ { "name": "Execute Pipeline", "description": "The lowest level executor with the metadata framework to call existing processing pipelines within Data Factory. The function called will block processing and wait for an outcome.", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Log Pipeline Running", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Pipeline Params", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get SPN Details", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "ExecutePipeline", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n\t\"tenantId\": \"',pipeline().parameters.TenantId,'\",\n\t\"applicationId\": \"',activity('Get SPN Details').output.firstRow.Id,'\",\n\t\"authenticationKey\": \"',activity('Get SPN Details').output.firstRow.Secret,'\",\n\t\"subscriptionId\": \"',pipeline().parameters.SubscriptionId,'\",\n\t\"resourceGroup\": \"',item().ResourceGroupName,'\",\n\t\"factoryName\": \"',item().DataFactoryName,'\",\n\t\"pipelineName\": \"',item().PipelineName,'\"',activity('Get Pipeline Params').output.firstRow.Params,'\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Get Pipeline Params", "description": "Returns any parameters from metadata required for the processing pipeline being called. The output can be an empty string if no parameters are required.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelineParameters]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@item().PipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Pipeline Running", "description": "Sets the current pipeline with a status of running within the current execution database table.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get SPN Details", "description": "Return the SPN ID and Secret for the processing pipeline being executed. Called at this level as each pipeline can have a different SPN.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetServicePrincipal]", "storedProcedureParameters": { "DataFactory": { "type": "String", "value": { "value": "@item().DataFactoryName", "type": "Expression" } }, "PipelineName": { "type": "String", "value": { "value": "@item().PipelineName", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Activity Failure", "description": "Handle true failures from calling out to the Azure Function and update the current execution table accordingly so a restart can occur.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" }, "CallingActivity": { "value": "ExecutePipeline", "type": "String" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Running Pipeline Handler", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "04-Infant", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "tenantId": { "value": "@pipeline().parameters.TenantId", "type": "Expression" }, "applicationId": { "value": "@activity('Get SPN Details').output.firstRow.Id", "type": "Expression" }, "authenticationKey": { "value": "@activity('Get SPN Details').output.firstRow.Secret", "type": "Expression" }, "subscriptionId": { "value": "@pipeline().parameters.SubscriptionId", "type": "Expression" }, "resourceGroup": { "value": "@item().ResourceGroupName", "type": "Expression" }, "factoryName": { "value": "@item().DataFactoryName", "type": "Expression" }, "pipelineName": { "value": "@item().PipelineName", "type": "Expression" }, "runId": { "value": "@activity('Execute Pipeline').output.RunId", "type": "Expression" }, "executionId": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "stageId": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "pipelineId": { "value": "@item().PipelineId", "type": "Expression" } } } }, { "name": "Set Run Id", "description": "Provide the actual ADF run ID back to the current execution table for long term logging and alignment between the metadata other Azure monitoring tools.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunId]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@activity('Execute Pipeline').output.RunId", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check For Alerts", "description": "Checks the properties tables and if any recipients in the database require alerts sending for the current pipeline ID.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[CheckForEmailAlerts]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@item().PipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Send Alerts", "description": "True = alerts need sending.\nFalse = do nothing.", "type": "IfCondition", "dependsOn": [ { "activity": "Set Run Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Check For Alerts", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Running Pipeline Handler", "dependencyConditions": [ "Completed" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@activity('Check For Alerts').output.firstRow.SendAlerts", "type": "Expression" }, "ifTrueActivities": [ { "name": "Get Email Parts", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetEmailAlertParts]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@item().PipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Send Email", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Get Email Parts", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "SendEmail", "method": "POST", "headers": {}, "body": { "value": "@activity('Get Email Parts').output.firstRow", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } } ] } } ] } } ], "parameters": { "StageId": { "type": "int" }, "ExecutionId": { "type": "string" }, "TenantId": { "type": "string" }, "SubscriptionId": { "type": "string" } }, "variables": { "FunctionBody": { "type": "String" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/FrameworkFunctions')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", "[concat(variables('factoryId'), '/pipelines/04-Infant')]" ] }, { "name": "[concat(parameters('factoryName'), '/04-Infant')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk infant pipeline used to check when the processing pipeline called by the Child completes and passes the resulting status back to the metadata database.", "activities": [ { "name": "Wait Until Pipeline Completes", "description": "Loops until the pipeline called completes.\n\nSimple status:\n- Running = new iteration.\n- Done = break.", "type": "Until", "dependsOn": [ { "activity": "Get Wait Duration", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@equals('Done',activity('Get Pipeline Status').output.SimpleStatus)", "type": "Expression" }, "activities": [ { "name": "Get Pipeline Status", "description": "Checks the status of a given processing pipeline and provides the value for the downstream framework activities to act upon.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "CheckPipelineStatus", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',pipeline().parameters.TenantId,'\",\n \"applicationId\": \"',pipeline().parameters.applicationId,'\",\n \"authenticationKey\": \"',pipeline().parameters.authenticationKey,'\",\n \"subscriptionId\": \"',pipeline().parameters.subscriptionId,'\",\n \"resourceGroup\": \"',pipeline().parameters.resourceGroup,'\",\n \"factoryName\": \"',pipeline().parameters.factoryName,'\",\n \"pipelineName\": \"',pipeline().parameters.pipelineName,'\",\n \"runId\": \"',pipeline().parameters.runId,'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Wait If Running", "description": "True = Do nothing.\nFalse = Wait.", "type": "IfCondition", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@equals('Done',activity('Get Pipeline Status').output.SimpleStatus)", "type": "Expression" }, "ifFalseActivities": [ { "name": "Wait for Pipeline", "description": "The processing pipeline is still running so Wait before checking its status again.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@activity('Get Wait Duration').output.firstRow.PropertyValue", "type": "Expression" } } } ] } }, { "name": "Set Last Update", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineLastStatusCheck]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Log Activity Failure", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "CallingActivity": { "value": "GetPipelineStatus", "type": "String" }, "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ], "timeout": "7.00:00:00" } }, { "name": "Pipeline Result", "description": "Receives the outcome from the function execution for a given processing pipeline and updates the current execution table with different pipelines status values depending on the result (case).", "type": "Switch", "dependsOn": [ { "activity": "Wait Until Pipeline Completes", "dependencyConditions": [ "Completed" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@activity('Get Pipeline Status').output.Status", "type": "Expression" }, "cases": [ { "value": "Succeeded", "activities": [ { "name": "Pipeline Status Succeeded", "description": "Updates the current execution table with a pipeline status of success if the function outcome is succeeded.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Failed", "activities": [ { "name": "Pipeline Status Failed", "description": "Updates the current execution table with a pipeline status of failed if the function outcome is failed. Also blocks pipelines in the downstream execution stage.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@activity('Get Pipeline Status').output.RunId", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Error Details", "description": "Get the activity error details for the run ID of the worker pipeline called. Returns an array of all errors.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "GetActivityErrors", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',pipeline().parameters.TenantId,'\",\n \"applicationId\": \"',pipeline().parameters.applicationId,'\",\n \"authenticationKey\": \"',pipeline().parameters.authenticationKey,'\",\n \"subscriptionId\": \"',pipeline().parameters.subscriptionId,'\",\n \"resourceGroup\": \"',pipeline().parameters.resourceGroup,'\",\n \"factoryName\": \"',pipeline().parameters.factoryName,'\",\n \"pipelineName\": \"',pipeline().parameters.pipelineName,'\",\n \"runId\": \"',pipeline().parameters.runId,'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Log Error Details", "description": "Parses pipeline error details and persists them to the metadata database error log table.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Error Details", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetErrorLogDetails]", "storedProcedureParameters": { "JsonErrorDetails": { "value": { "value": "@string(activity('Get Error Details').output)", "type": "Expression" }, "type": "String" }, "LocalExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } ], "defaultActivities": [ { "name": "Pipeline Status Unknown", "description": "Updates the current execution table with a pipeline status of unknown if the function returns an unexpected outcome.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineUnknown]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Get Wait Duration", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "PipelineStatusCheckDuration" } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } } ], "parameters": { "tenantId": { "type": "string" }, "applicationId": { "type": "string" }, "authenticationKey": { "type": "string" }, "subscriptionId": { "type": "string" }, "resourceGroup": { "type": "string" }, "factoryName": { "type": "string" }, "pipelineName": { "type": "string" }, "runId": { "type": "string" }, "executionId": { "type": "string" }, "stageId": { "type": "int" }, "pipelineId": { "type": "int" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/FrameworkFunctions')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/Intentional Error')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } }, { "name": "Raise Errors or Not", "type": "IfCondition", "dependsOn": [ { "activity": "Wait1", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@equals(pipeline().parameters.RaiseErrors,'true')", "type": "Expression" }, "ifTrueActivities": [ { "name": "Call Fail Procedure", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[dbo].[FailProcedure]", "storedProcedureParameters": { "RaiseError": { "value": { "value": "@pipeline().parameters.RaiseErrors", "type": "Expression" }, "type": "String" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Call Fail Notebook", "type": "DatabricksNotebook", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "notebookPath": "/Playground/Throw Exception", "baseParameters": { "RaiseError": { "value": "@pipeline().parameters.RaiseErrors", "type": "Expression" } } }, "linkedServiceName": { "referenceName": "BricksOfData", "type": "LinkedServiceReference" } } ] } } ], "parameters": { "RaiseErrors": { "type": "string", "defaultValue": "false" }, "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", "[concat(variables('factoryId'), '/linkedServices/BricksOfData')]" ] }, { "name": "[concat(parameters('factoryName'), '/Wait 1')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 10')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait10", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 2')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait2", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 3')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait3", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 4')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait4", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 5')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait5", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 6')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait6", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 7')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait7", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 8')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait8", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 9')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait9", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 90 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/GetSetMetadata')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" }, "annotations": [ "ADF.procfwk" ], "type": "AzureSqlTable", "schema": [], "typeProperties": {} }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/BricksOfData')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [], "type": "AzureDatabricks", "typeProperties": { "domain": "https://northeurope.azuredatabricks.net", "accessToken": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "BricksOfDataToken", "secretVersion": "" }, "existingClusterId": "0422-090117-slots899" } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/FrameworkFunctions')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [ "ADF.procfwk" ], "type": "AzureFunction", "typeProperties": { "functionAppUrl": "https://frameworksupportfunctions.azurewebsites.net", "functionKey": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "FrameworkFunctionsKey" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/Keys')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection to Key Vault for all other ADF linked service credentials required to run the processing framework.", "annotations": [ "ADF.procfwk" ], "type": "AzureKeyVault", "typeProperties": { "baseUrl": "[parameters('Keys_properties_typeProperties_baseUrl')]" } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/SupportDatabase')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection between ADF and processing framework metadata SQLDB.", "annotations": [ "ADF.procfwk" ], "type": "AzureSqlDatabase", "typeProperties": { "connectionString": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "[parameters('SupportDatabase_properties_typeProperties_connectionString_secretName')]" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/FunctionalTestingTrigger')]", "type": "Microsoft.DataFactory/factories/triggers", "apiVersion": "2018-06-01", "properties": { "description": "Used for functional testing of the framework in a dedicated environment.", "annotations": [ "ADF.procfwk" ], "runtimeState": "Stopped", "pipelines": [ { "pipelineReference": { "referenceName": "01-Grandparent", "type": "PipelineReference" }, "parameters": {} } ], "type": "ScheduleTrigger", "typeProperties": { "recurrence": { "frequency": "Hour", "interval": 2, "startTime": "2020-04-06T15:00:00.000Z", "timeZone": "UTC" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/pipelines/01-Grandparent')]" ] } ] } ================================================ FILE: ARM Templates/Data Factory/v1.8 Export.json ================================================ { "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { "factoryName": { "type": "string", "metadata": "Data Factory name", "defaultValue": "" }, "FrameworkFunctions_properties_typeProperties_functionAppUrl": { "type": "string", "defaultValue": "" }, "Keys_properties_typeProperties_baseUrl": { "type": "string", "defaultValue": "" }, "SupportDatabase_properties_typeProperties_connectionString_secretName": { "type": "string", "defaultValue": "" } }, "variables": { "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" }, "resources": [ { "name": "[concat(parameters('factoryName'), '/01-Grandparent')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk grandparent pipeline used optionally to bootstrap any wider processes in your Data Factory that then calls the processing framework.", "activities": [ { "name": "Framework Processing", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Set Random Waits", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "02-Parent", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": {} } }, { "name": "Set Random Waits", "description": "For functional testing only.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[dbo].[SetRandomWaitValues]" }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ], "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/pipelines/02-Parent')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/02-Parent')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk parent pipeline used to bootstrap the orchestration framework in perform the first level ForEach calls in sequence for the metadata stages.", "activities": [ { "name": "Get Stages", "description": "Returns a distinct list of execution stages within the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Execution Wrapper", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetStages]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Stages", "description": "Top level ForEach to sequentially call all processing stages within the framework metadata. Items for iteration passed from the Get Stages lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Stages", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Tenant Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Subscription Id", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Stages').output.value", "type": "Expression" }, "isSequential": true, "activities": [ { "name": "Stage Executor", "description": "Call to the framework generic child pipeline for a given execution stage.", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Log Stage Preparing", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "03-Child", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "StageId": { "value": "@item().StageId", "type": "Expression" }, "ExecutionId": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" }, "TenantId": { "value": "@activity('Get Tenant Id').output.firstRow.PropertyValue", "type": "Expression" }, "SubscriptionId": { "value": "@activity('Get Subscription Id').output.firstRow.PropertyValue", "type": "Expression" } } } }, { "name": "Log Stage Preparing", "description": "Update the current execution table flagging all pipelines within the stage as preparing.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Check and Update Blockers", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogStagePreparing]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check and Update Blockers", "description": "Used to double check and stop the next execution stage if failures and blockers have be incurred. This also depends on the failure handling property value which defines the stored procedure behaviour.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[CheckForBlockedPipelines]", "storedProcedureParameters": { "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Execution Wrapper", "description": "Wrapper to reset and restart processing or create a completely new execution instance of the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Clean Up Previous Run", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[ExecutionWrapper]", "storedProcedureParameters": { "CallingDataFactory": { "type": "String", "value": { "value": "@pipeline().DataFactory", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Check Outcome and Update Logs", "description": "After a successful execution run the current execution metadata is moved to the long term logging table by this stored procedure call. Otherwise an error will be raised.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Stages", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[UpdateExecutionLog]" }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Tenant Id", "description": "Returning the Azure Tenant Id from the metadata properties table.", "type": "Lookup", "dependsOn": [ { "activity": "Metadata Integrity Checks", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "TenantId" } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Get Subscription Id", "description": "Returning the Azure Subscription Id from the metadata properties table.", "type": "Lookup", "dependsOn": [ { "activity": "Metadata Integrity Checks", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "SubscriptionId" } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Metadata Integrity Checks", "description": "Performs a series of checks on all metadata held in the framework SQLDB. This is intended to raise errors before an execution run even starts.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[CheckMetadataIntegrity]", "storedProcedureParameters": { "DebugMode": { "type": "Boolean", "value": "false" } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Clean Up Previous Run", "description": "Handle Worker pipelines that are reported as Running when the parent pipeline is called again. Get what the actual status of those pipelines is.", "type": "ForEach", "dependsOn": [ { "activity": "Metadata Integrity Checks", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Metadata Integrity Checks').output.value", "type": "Expression" }, "isSequential": false, "activities": [ { "name": "Get SPN Details", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetServicePrincipal]", "storedProcedureParameters": { "DataFactory": { "type": "String", "value": { "value": "@item().DataFactoryName", "type": "Expression" } }, "PipelineName": { "type": "String", "value": { "value": "@item().PipelineName", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Pipeline Checking", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineChecking]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Pipeline Status", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Get SPN Details", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Log Pipeline Checking", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "CheckPipelineStatus", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',item().TenantId,'\",\n \"applicationId\": \"',activity('Get SPN Details').output.firstRow.Id,'\",\n \"authenticationKey\": \"',activity('Get SPN Details').output.firstRow.Secret,'\",\n \"subscriptionId\": \"',item().SubscriptionId,'\",\n \"resourceGroup\": \"',item().ResourceGroupName,'\",\n \"factoryName\": \"',item().DataFactoryName,'\",\n \"pipelineName\": \"',item().PipelineName,'\",\n \"runId\": \"',item().AdfPipelineRunId,'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Set Pipeline Status", "type": "Switch", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@activity('Get Pipeline Status').output.Status", "type": "Expression" }, "cases": [ { "value": "Failed", "activities": [ { "name": "Pipeline Status Failed", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": null, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Succeeded", "activities": [ { "name": "Pipeline Status Succeeded", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Queued", "activities": [ { "name": "Pipeline Status Queued - Running", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "InProgress", "activities": [ { "name": "Pipeline Status InProgress - Running", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } ], "defaultActivities": [ { "name": "Pipeline Status Unknown", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineUnknown]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Set Last Check DateTime", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineLastStatusCheck]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } } ], "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", "[concat(variables('factoryId'), '/pipelines/03-Child')]", "[concat(variables('factoryId'), '/linkedServices/FrameworkFunctions')]" ] }, { "name": "[concat(parameters('factoryName'), '/03-Child')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk child pipeline used to execute Worker pipelines within a given execution stage. This pipeline will be called once for each stage, then execute all Workers in parallel.", "activities": [ { "name": "Get Pipelines", "description": "Returns all pipelines from the metadata to be executed within a given processing stage.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelinesInStage]", "storedProcedureParameters": { "StageId": { "type": "Int32", "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Pipelines", "description": "Second level ForEach to run in parallel all pipelines within the stage. Items for iteration passed from the Get Pipelines lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Pipelines", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Pipelines').output.value", "type": "Expression" }, "isSequential": false, "batchCount": 40, "activities": [ { "name": "Execute Pipeline", "description": "The lowest level executor with the metadata framework to call existing processing pipelines within Data Factory. The function called will block processing and wait for an outcome.", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Log Pipeline Running", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Pipeline Params", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get SPN Details", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "ExecutePipeline", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n\t\"tenantId\": \"',pipeline().parameters.TenantId,'\",\n\t\"applicationId\": \"',activity('Get SPN Details').output.firstRow.Id,'\",\n\t\"authenticationKey\": \"',activity('Get SPN Details').output.firstRow.Secret,'\",\n\t\"subscriptionId\": \"',pipeline().parameters.SubscriptionId,'\",\n\t\"resourceGroup\": \"',item().ResourceGroupName,'\",\n\t\"factoryName\": \"',item().DataFactoryName,'\",\n\t\"pipelineName\": \"',item().PipelineName,'\"',activity('Get Pipeline Params').output.firstRow.Params,'\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Get Pipeline Params", "description": "Returns any parameters from metadata required for the processing pipeline being called. The output can be an empty string if no parameters are required.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelineParameters]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@item().PipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Pipeline Running", "description": "Sets the current pipeline with a status of running within the current execution database table.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get SPN Details", "description": "Return the SPN ID and Secret for the processing pipeline being executed. Called at this level as each pipeline can have a different SPN.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetServicePrincipal]", "storedProcedureParameters": { "DataFactory": { "type": "String", "value": { "value": "@item().DataFactoryName", "type": "Expression" } }, "PipelineName": { "type": "String", "value": { "value": "@item().PipelineName", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Activity Failure", "description": "Handle true failures from calling out to the Azure Function and update the current execution table accordingly so a restart can occur.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" }, "CallingActivity": { "value": "ExecutePipeline", "type": "String" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Running Pipeline Handler", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "04-Infant", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "tenantId": { "value": "@pipeline().parameters.TenantId", "type": "Expression" }, "applicationId": { "value": "@activity('Get SPN Details').output.firstRow.Id", "type": "Expression" }, "authenticationKey": { "value": "@activity('Get SPN Details').output.firstRow.Secret", "type": "Expression" }, "subscriptionId": { "value": "@pipeline().parameters.SubscriptionId", "type": "Expression" }, "resourceGroup": { "value": "@item().ResourceGroupName", "type": "Expression" }, "factoryName": { "value": "@item().DataFactoryName", "type": "Expression" }, "pipelineName": { "value": "@item().PipelineName", "type": "Expression" }, "runId": { "value": "@activity('Execute Pipeline').output.RunId", "type": "Expression" }, "executionId": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "stageId": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "pipelineId": { "value": "@item().PipelineId", "type": "Expression" } } } }, { "name": "Set Run Id", "description": "Provide the actual ADF run ID back to the current execution table for long term logging and alignment between the metadata other Azure monitoring tools.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunId]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@activity('Execute Pipeline').output.RunId", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check For Alerts", "description": "Checks the properties tables and if any recipients in the database require alerts sending for the current pipeline ID.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[CheckForEmailAlerts]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@item().PipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Send Alerts", "description": "True = alerts need sending.\nFalse = do nothing.", "type": "IfCondition", "dependsOn": [ { "activity": "Set Run Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Check For Alerts", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Running Pipeline Handler", "dependencyConditions": [ "Completed" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@activity('Check For Alerts').output.firstRow.SendAlerts", "type": "Expression" }, "ifTrueActivities": [ { "name": "Get Email Parts", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetEmailAlertParts]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@item().PipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Send Email", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Get Email Parts", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "SendEmail", "method": "POST", "headers": {}, "body": { "value": "@activity('Get Email Parts').output.firstRow", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } } ] } } ] } } ], "parameters": { "StageId": { "type": "int" }, "ExecutionId": { "type": "string" }, "TenantId": { "type": "string" }, "SubscriptionId": { "type": "string" } }, "variables": { "FunctionBody": { "type": "String" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/FrameworkFunctions')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", "[concat(variables('factoryId'), '/pipelines/04-Infant')]" ] }, { "name": "[concat(parameters('factoryName'), '/04-Infant')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk infant pipeline used to check when the processing pipeline called by the Child completes and passes the resulting status back to the metadata database.", "activities": [ { "name": "Wait Until Pipeline Completes", "description": "Loops until the pipeline called completes.\n\nSimple status:\n- Running = new iteration.\n- Done = break.", "type": "Until", "dependsOn": [ { "activity": "Get Wait Duration", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@equals('Done',activity('Get Pipeline Status').output.SimpleStatus)", "type": "Expression" }, "activities": [ { "name": "Get Pipeline Status", "description": "Checks the status of a given processing pipeline and provides the value for the downstream framework activities to act upon.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "CheckPipelineStatus", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',pipeline().parameters.TenantId,'\",\n \"applicationId\": \"',pipeline().parameters.applicationId,'\",\n \"authenticationKey\": \"',pipeline().parameters.authenticationKey,'\",\n \"subscriptionId\": \"',pipeline().parameters.subscriptionId,'\",\n \"resourceGroup\": \"',pipeline().parameters.resourceGroup,'\",\n \"factoryName\": \"',pipeline().parameters.factoryName,'\",\n \"pipelineName\": \"',pipeline().parameters.pipelineName,'\",\n \"runId\": \"',pipeline().parameters.runId,'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Wait If Running", "description": "True = Do nothing.\nFalse = Wait.", "type": "IfCondition", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@equals('Done',activity('Get Pipeline Status').output.SimpleStatus)", "type": "Expression" }, "ifFalseActivities": [ { "name": "Wait for Pipeline", "description": "The processing pipeline is still running so Wait before checking its status again.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@activity('Get Wait Duration').output.firstRow.PropertyValue", "type": "Expression" } } } ] } }, { "name": "Set Last Check DateTime", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineLastStatusCheck]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Log Activity Failure", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "CallingActivity": { "value": "GetPipelineStatus", "type": "String" }, "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ], "timeout": "7.00:00:00" } }, { "name": "Set Pipeline Result", "description": "Receives the outcome from the function execution for a given processing pipeline and updates the current execution table with different pipelines status values depending on the result (case).", "type": "Switch", "dependsOn": [ { "activity": "Wait Until Pipeline Completes", "dependencyConditions": [ "Completed" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@activity('Get Pipeline Status').output.Status", "type": "Expression" }, "cases": [ { "value": "Succeeded", "activities": [ { "name": "Pipeline Status Succeeded", "description": "Updates the current execution table with a pipeline status of success if the function outcome is succeeded.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Failed", "activities": [ { "name": "Pipeline Status Failed", "description": "Updates the current execution table with a pipeline status of failed if the function outcome is failed. Also blocks pipelines in the downstream execution stage.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@activity('Get Pipeline Status').output.RunId", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Error Details", "description": "Get the activity error details for the run ID of the worker pipeline called. Returns an array of all errors.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "GetActivityErrors", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',pipeline().parameters.TenantId,'\",\n \"applicationId\": \"',pipeline().parameters.applicationId,'\",\n \"authenticationKey\": \"',pipeline().parameters.authenticationKey,'\",\n \"subscriptionId\": \"',pipeline().parameters.subscriptionId,'\",\n \"resourceGroup\": \"',pipeline().parameters.resourceGroup,'\",\n \"factoryName\": \"',pipeline().parameters.factoryName,'\",\n \"pipelineName\": \"',pipeline().parameters.pipelineName,'\",\n \"runId\": \"',pipeline().parameters.runId,'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Log Error Details", "description": "Parses pipeline error details and persists them to the metadata database error log table.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Error Details", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetErrorLogDetails]", "storedProcedureParameters": { "JsonErrorDetails": { "value": { "value": "@string(activity('Get Error Details').output)", "type": "Expression" }, "type": "String" }, "LocalExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } ], "defaultActivities": [ { "name": "Pipeline Status Unknown", "description": "Updates the current execution table with a pipeline status of unknown if the function returns an unexpected outcome.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineUnknown]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Get Wait Duration", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "PipelineStatusCheckDuration" } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } } ], "parameters": { "tenantId": { "type": "string" }, "applicationId": { "type": "string" }, "authenticationKey": { "type": "string" }, "subscriptionId": { "type": "string" }, "resourceGroup": { "type": "string" }, "factoryName": { "type": "string" }, "pipelineName": { "type": "string" }, "runId": { "type": "string" }, "executionId": { "type": "string" }, "stageId": { "type": "int" }, "pipelineId": { "type": "int" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/FrameworkFunctions')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/Intentional Error')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } }, { "name": "Raise Errors or Not", "type": "IfCondition", "dependsOn": [ { "activity": "Wait1", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@equals(pipeline().parameters.RaiseErrors,'true')", "type": "Expression" }, "ifTrueActivities": [ { "name": "Call Fail Procedure", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[dbo].[FailProcedure]", "storedProcedureParameters": { "RaiseError": { "value": { "value": "@pipeline().parameters.RaiseErrors", "type": "Expression" }, "type": "String" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } } ], "parameters": { "RaiseErrors": { "type": "string", "defaultValue": "false" }, "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/Wait 1')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 10')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait10", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 2')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait2", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 3')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait3", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 4')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait4", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 5')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait5", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 6')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait6", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 7')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait7", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 8')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait8", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int" } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 9')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait9", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 15 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/GetSetMetadata')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" }, "annotations": [ "ADF.procfwk" ], "type": "AzureSqlTable", "schema": [], "typeProperties": {} }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/FrameworkFunctions')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [ "ADF.procfwk" ], "type": "AzureFunction", "typeProperties": { "functionAppUrl": "[parameters('FrameworkFunctions_properties_typeProperties_functionAppUrl')]", "functionKey": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "FrameworkFunctionsKey" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/Keys')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection to Key Vault for all other ADF linked service credentials required to run the processing framework.", "annotations": [ "ADF.procfwk" ], "type": "AzureKeyVault", "typeProperties": { "baseUrl": "[parameters('Keys_properties_typeProperties_baseUrl')]" } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/SupportDatabase')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection between ADF and processing framework metadata SQLDB.", "annotations": [ "ADF.procfwk" ], "type": "AzureSqlDatabase", "typeProperties": { "connectionString": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "[parameters('SupportDatabase_properties_typeProperties_connectionString_secretName')]" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/FunctionalTestingTrigger')]", "type": "Microsoft.DataFactory/factories/triggers", "apiVersion": "2018-06-01", "properties": { "description": "Used for functional testing of the framework in a dedicated environment.", "annotations": [ "ADF.procfwk" ], "runtimeState": "Stopped", "pipelines": [ { "pipelineReference": { "referenceName": "01-Grandparent", "type": "PipelineReference" }, "parameters": {} } ], "type": "ScheduleTrigger", "typeProperties": { "recurrence": { "frequency": "Hour", "interval": 2, "startTime": "2020-04-06T15:00:00.000Z", "timeZone": "UTC" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/pipelines/01-Grandparent')]" ] } ] } ================================================ FILE: ARM Templates/Data Factory/v1.8.3 Export.json ================================================ { "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { "factoryName": { "type": "string", "metadata": "Data Factory name", "defaultValue": "FrameworkFactory" }, "FrameworkFunctions_properties_typeProperties_functionAppUrl": { "type": "string", "defaultValue": "" }, "Keys_properties_typeProperties_baseUrl": { "type": "string", "defaultValue": "" }, "SupportDatabase_properties_typeProperties_connectionString_secretName": { "type": "string", "defaultValue": "" } }, "variables": { "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" }, "resources": [ { "name": "[concat(parameters('factoryName'), '/01-Grandparent')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk grandparent pipeline used optionally to bootstrap any wider processes in your Data Factory that then calls the processing framework.", "activities": [ { "name": "Framework Processing", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Set Random Waits", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "02-Parent", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": {} } }, { "name": "Set Random Waits", "description": "For functional testing only.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[dbo].[SetRandomWaitValues]" }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ], "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/pipelines/02-Parent')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/02-Parent')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk parent pipeline used to bootstrap the orchestration framework in perform the first level ForEach calls in sequence for the metadata stages.", "activities": [ { "name": "Get Stages", "description": "Returns a distinct list of execution stages within the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Execution Wrapper", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetStages]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Stages", "description": "Top level ForEach to sequentially call all processing stages within the framework metadata. Items for iteration passed from the Get Stages lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Stages", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Tenant Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Subscription Id", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Stages').output.value", "type": "Expression" }, "isSequential": true, "activities": [ { "name": "Stage Executor", "description": "Call to the framework generic child pipeline for a given execution stage.", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Log Stage Preparing", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "03-Child", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "StageId": { "value": "@item().StageId", "type": "Expression" }, "ExecutionId": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" }, "TenantId": { "value": "@activity('Get Tenant Id').output.firstRow.PropertyValue", "type": "Expression" }, "SubscriptionId": { "value": "@activity('Get Subscription Id').output.firstRow.PropertyValue", "type": "Expression" } } } }, { "name": "Log Stage Preparing", "description": "Update the current execution table flagging all pipelines within the stage as preparing.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Check and Update Blockers", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogStagePreparing]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check and Update Blockers", "description": "Used to double check and stop the next execution stage if failures and blockers have be incurred. This also depends on the failure handling property value which defines the stored procedure behaviour.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[CheckForBlockedPipelines]", "storedProcedureParameters": { "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Execution Wrapper", "description": "Wrapper to reset and restart processing or create a completely new execution instance of the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Clean Up Previous Run", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[ExecutionWrapper]", "storedProcedureParameters": { "CallingDataFactory": { "type": "String", "value": { "value": "@pipeline().DataFactory", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Check Outcome and Update Logs", "description": "After a successful execution run the current execution metadata is moved to the long term logging table by this stored procedure call. Otherwise an error will be raised.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Stages", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[UpdateExecutionLog]" }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Tenant Id", "description": "Returning the Azure Tenant Id from the metadata properties table.", "type": "Lookup", "dependsOn": [ { "activity": "Metadata Integrity Checks", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "TenantId" } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Get Subscription Id", "description": "Returning the Azure Subscription Id from the metadata properties table.", "type": "Lookup", "dependsOn": [ { "activity": "Metadata Integrity Checks", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "SubscriptionId" } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Metadata Integrity Checks", "description": "Performs a series of checks on all metadata held in the framework SQLDB. This is intended to raise errors before an execution run even starts.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[CheckMetadataIntegrity]", "storedProcedureParameters": { "DebugMode": { "type": "Boolean", "value": "false" } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Clean Up Previous Run", "description": "Handle Worker pipelines that are reported as Running when the parent pipeline is called again. Get what the actual status of those pipelines is.", "type": "ForEach", "dependsOn": [ { "activity": "Metadata Integrity Checks", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Metadata Integrity Checks').output.value", "type": "Expression" }, "isSequential": false, "activities": [ { "name": "Get SPN Details", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetServicePrincipal]", "storedProcedureParameters": { "DataFactory": { "type": "String", "value": { "value": "@item().DataFactoryName", "type": "Expression" } }, "PipelineName": { "type": "String", "value": { "value": "@item().PipelineName", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Pipeline Checking", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineChecking]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Pipeline Status", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Get SPN Details", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Log Pipeline Checking", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "CheckPipelineStatus", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',item().TenantId,'\",\n \"applicationId\": \"',activity('Get SPN Details').output.firstRow.Id,'\",\n \"authenticationKey\": \"',activity('Get SPN Details').output.firstRow.Secret,'\",\n \"subscriptionId\": \"',item().SubscriptionId,'\",\n \"resourceGroup\": \"',item().ResourceGroupName,'\",\n \"factoryName\": \"',item().DataFactoryName,'\",\n \"pipelineName\": \"',item().PipelineName,'\",\n \"runId\": \"',item().AdfPipelineRunId,'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Set Pipeline Status", "type": "Switch", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@activity('Get Pipeline Status').output.Status", "type": "Expression" }, "cases": [ { "value": "Failed", "activities": [ { "name": "Pipeline Status Failed", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": null, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Succeeded", "activities": [ { "name": "Pipeline Status Succeeded", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Queued", "activities": [ { "name": "Pipeline Status Queued - Running", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "InProgress", "activities": [ { "name": "Pipeline Status InProgress - Running", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Cancelled", "activities": [ { "name": "Pipeline Status Cancelled", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineCancelled]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } ], "defaultActivities": [ { "name": "Pipeline Status Unknown", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineUnknown]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Set Last Check DateTime", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineLastStatusCheck]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } } ], "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", "[concat(variables('factoryId'), '/pipelines/03-Child')]", "[concat(variables('factoryId'), '/linkedServices/FrameworkFunctions')]" ] }, { "name": "[concat(parameters('factoryName'), '/03-Child')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk child pipeline used to execute Worker pipelines within a given execution stage. This pipeline will be called once for each stage, then execute all Workers in parallel.", "activities": [ { "name": "Get Pipelines", "description": "Returns all pipelines from the metadata to be executed within a given processing stage.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelinesInStage]", "storedProcedureParameters": { "StageId": { "type": "Int32", "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Pipelines", "description": "Second level ForEach to run in parallel all pipelines within the stage. Items for iteration passed from the Get Pipelines lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Pipelines", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Pipelines').output.value", "type": "Expression" }, "isSequential": false, "batchCount": 40, "activities": [ { "name": "Execute Pipeline", "description": "The lowest level executor with the metadata framework to call existing processing pipelines within Data Factory. The function called will block processing and wait for an outcome.", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Log Pipeline Running", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Pipeline Params", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get SPN Details", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "ExecutePipeline", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n\t\"tenantId\": \"',pipeline().parameters.TenantId,'\",\n\t\"applicationId\": \"',activity('Get SPN Details').output.firstRow.Id,'\",\n\t\"authenticationKey\": \"',activity('Get SPN Details').output.firstRow.Secret,'\",\n\t\"subscriptionId\": \"',pipeline().parameters.SubscriptionId,'\",\n\t\"resourceGroup\": \"',item().ResourceGroupName,'\",\n\t\"factoryName\": \"',item().DataFactoryName,'\",\n\t\"pipelineName\": \"',item().PipelineName,'\"',activity('Get Pipeline Params').output.firstRow.Params,'\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Get Pipeline Params", "description": "Returns any parameters from metadata required for the processing pipeline being called. The output can be an empty string if no parameters are required.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelineParameters]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@item().PipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Pipeline Running", "description": "Sets the current pipeline with a status of running within the current execution database table.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get SPN Details", "description": "Return the SPN ID and Secret for the processing pipeline being executed. Called at this level as each pipeline can have a different SPN.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetServicePrincipal]", "storedProcedureParameters": { "DataFactory": { "type": "String", "value": { "value": "@item().DataFactoryName", "type": "Expression" } }, "PipelineName": { "type": "String", "value": { "value": "@item().PipelineName", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Activity Failure", "description": "Handle true failures from calling out to the Azure Function and update the current execution table accordingly so a restart can occur.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" }, "CallingActivity": { "value": "ExecutePipeline", "type": "String" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Running Pipeline Handler", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "04-Infant", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "tenantId": { "value": "@pipeline().parameters.TenantId", "type": "Expression" }, "applicationId": { "value": "@activity('Get SPN Details').output.firstRow.Id", "type": "Expression" }, "authenticationKey": { "value": "@activity('Get SPN Details').output.firstRow.Secret", "type": "Expression" }, "subscriptionId": { "value": "@pipeline().parameters.SubscriptionId", "type": "Expression" }, "resourceGroup": { "value": "@item().ResourceGroupName", "type": "Expression" }, "factoryName": { "value": "@item().DataFactoryName", "type": "Expression" }, "pipelineName": { "value": "@item().PipelineName", "type": "Expression" }, "runId": { "value": "@activity('Execute Pipeline').output.RunId", "type": "Expression" }, "executionId": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "stageId": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "pipelineId": { "value": "@item().PipelineId", "type": "Expression" } } } }, { "name": "Set Run Id", "description": "Provide the actual ADF run ID back to the current execution table for long term logging and alignment between the metadata other Azure monitoring tools.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunId]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@activity('Execute Pipeline').output.RunId", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check For Alerts", "description": "Checks the properties tables and if any recipients in the database require alerts sending for the current pipeline ID.", "type": "Lookup", "dependsOn": [ { "activity": "Set Run Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Running Pipeline Handler", "dependencyConditions": [ "Completed" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[CheckForEmailAlerts]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@item().PipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Send Alerts", "description": "True = alerts need sending.\nFalse = do nothing.", "type": "IfCondition", "dependsOn": [ { "activity": "Check For Alerts", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@activity('Check For Alerts').output.firstRow.SendAlerts", "type": "Expression" }, "ifTrueActivities": [ { "name": "Get Email Parts", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetEmailAlertParts]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@item().PipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Send Email", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Get Email Parts", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "SendEmail", "method": "POST", "headers": {}, "body": { "value": "@activity('Get Email Parts').output.firstRow", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } } ] } } ] } } ], "parameters": { "StageId": { "type": "int" }, "ExecutionId": { "type": "string" }, "TenantId": { "type": "string" }, "SubscriptionId": { "type": "string" } }, "variables": { "FunctionBody": { "type": "String" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/FrameworkFunctions')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", "[concat(variables('factoryId'), '/pipelines/04-Infant')]" ] }, { "name": "[concat(parameters('factoryName'), '/04-Infant')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk infant pipeline used to check when the processing pipeline called by the Child completes and passes the resulting status back to the metadata database.", "activities": [ { "name": "Wait Until Pipeline Completes", "description": "Loops until the pipeline called completes.\n\nSimple status:\n- Running = new iteration.\n- Done = break.", "type": "Until", "dependsOn": [ { "activity": "Get Wait Duration", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@equals('Done',activity('Get Pipeline Status').output.SimpleStatus)", "type": "Expression" }, "activities": [ { "name": "Get Pipeline Status", "description": "Checks the status of a given processing pipeline and provides the value for the downstream framework activities to act upon.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "CheckPipelineStatus", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',pipeline().parameters.TenantId,'\",\n \"applicationId\": \"',pipeline().parameters.applicationId,'\",\n \"authenticationKey\": \"',pipeline().parameters.authenticationKey,'\",\n \"subscriptionId\": \"',pipeline().parameters.subscriptionId,'\",\n \"resourceGroup\": \"',pipeline().parameters.resourceGroup,'\",\n \"factoryName\": \"',pipeline().parameters.factoryName,'\",\n \"pipelineName\": \"',pipeline().parameters.pipelineName,'\",\n \"runId\": \"',pipeline().parameters.runId,'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Wait If Running", "description": "True = Do nothing.\nFalse = Wait.", "type": "IfCondition", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@equals('Done',activity('Get Pipeline Status').output.SimpleStatus)", "type": "Expression" }, "ifFalseActivities": [ { "name": "Wait for Pipeline", "description": "The processing pipeline is still running so Wait before checking its status again.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@activity('Get Wait Duration').output.firstRow.PropertyValue", "type": "Expression" } } } ] } }, { "name": "Set Last Check DateTime", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineLastStatusCheck]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Log Activity Failure", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "CallingActivity": { "value": "GetPipelineStatus", "type": "String" }, "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ], "timeout": "7.00:00:00" } }, { "name": "Set Pipeline Result", "description": "Receives the outcome from the function execution for a given processing pipeline and updates the current execution table with different pipelines status values depending on the result (case).", "type": "Switch", "dependsOn": [ { "activity": "Wait Until Pipeline Completes", "dependencyConditions": [ "Completed" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@activity('Get Pipeline Status').output.Status", "type": "Expression" }, "cases": [ { "value": "Succeeded", "activities": [ { "name": "Pipeline Status Succeeded", "description": "Updates the current execution table with a pipeline status of success if the function outcome is succeeded.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Failed", "activities": [ { "name": "Pipeline Status Failed", "description": "Updates the current execution table with a pipeline status of failed if the function outcome is failed. Also blocks pipelines in the downstream execution stage.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@activity('Get Pipeline Status').output.RunId", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Error Details", "description": "Get the activity error details for the run ID of the worker pipeline called. Returns an array of all errors.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "GetActivityErrors", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',pipeline().parameters.TenantId,'\",\n \"applicationId\": \"',pipeline().parameters.applicationId,'\",\n \"authenticationKey\": \"',pipeline().parameters.authenticationKey,'\",\n \"subscriptionId\": \"',pipeline().parameters.subscriptionId,'\",\n \"resourceGroup\": \"',pipeline().parameters.resourceGroup,'\",\n \"factoryName\": \"',pipeline().parameters.factoryName,'\",\n \"pipelineName\": \"',pipeline().parameters.pipelineName,'\",\n \"runId\": \"',pipeline().parameters.runId,'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Log Error Details", "description": "Parses pipeline error details and persists them to the metadata database error log table.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Error Details", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetErrorLogDetails]", "storedProcedureParameters": { "JsonErrorDetails": { "value": { "value": "@string(activity('Get Error Details').output)", "type": "Expression" }, "type": "String" }, "LocalExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Cancelled", "activities": [ { "name": "Pipeline Status Cancelled", "description": "Updates the current execution table with a pipeline status of unknown if the function returns an unexpected outcome.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineCancelled]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } ], "defaultActivities": [ { "name": "Pipeline Status Unknown", "description": "Updates the current execution table with a pipeline status of unknown if the function returns an unexpected outcome.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineUnknown]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Get Wait Duration", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "PipelineStatusCheckDuration" } }, "queryTimeout": "02:00:00" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } } ], "parameters": { "tenantId": { "type": "string" }, "applicationId": { "type": "string" }, "authenticationKey": { "type": "string" }, "subscriptionId": { "type": "string" }, "resourceGroup": { "type": "string" }, "factoryName": { "type": "string" }, "pipelineName": { "type": "string" }, "runId": { "type": "string" }, "executionId": { "type": "string" }, "stageId": { "type": "int" }, "pipelineId": { "type": "int" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/FrameworkFunctions')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/Intentional Error')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } }, { "name": "Raise Errors or Not", "type": "IfCondition", "dependsOn": [ { "activity": "Wait1", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@equals(pipeline().parameters.RaiseErrors,'true')", "type": "Expression" }, "ifTrueActivities": [ { "name": "Call Fail Procedure", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[dbo].[FailProcedure]", "storedProcedureParameters": { "RaiseError": { "value": { "value": "@pipeline().parameters.RaiseErrors", "type": "Expression" }, "type": "String" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } } ], "parameters": { "RaiseErrors": { "type": "string", "defaultValue": "false" }, "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/Wait 1')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 10')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait10", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 2')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait2", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 3')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait3", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 4')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait4", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 5')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait5", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 6')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait6", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 7')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait7", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 8')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait8", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 9')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait9", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 15 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/GetSetMetadata')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" }, "annotations": [ "ADF.procfwk" ], "type": "AzureSqlTable", "schema": [], "typeProperties": {} }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/FrameworkFunctions')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [ "ADF.procfwk" ], "type": "AzureFunction", "typeProperties": { "functionAppUrl": "[parameters('FrameworkFunctions_properties_typeProperties_functionAppUrl')]", "functionKey": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "FrameworkFunctionsKey" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/Keys')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection to Key Vault for all other ADF linked service credentials required to run the processing framework.", "annotations": [ "ADF.procfwk" ], "type": "AzureKeyVault", "typeProperties": { "baseUrl": "[parameters('Keys_properties_typeProperties_baseUrl')]" } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/SupportDatabase')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection between ADF and processing framework metadata SQLDB.", "annotations": [ "ADF.procfwk" ], "type": "AzureSqlDatabase", "typeProperties": { "connectionString": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "[parameters('SupportDatabase_properties_typeProperties_connectionString_secretName')]" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/FunctionalTestingTrigger')]", "type": "Microsoft.DataFactory/factories/triggers", "apiVersion": "2018-06-01", "properties": { "description": "Used for functional testing of the framework in a dedicated environment.", "annotations": [ "ADF.procfwk" ], "runtimeState": "Stopped", "pipelines": [ { "pipelineReference": { "referenceName": "01-Grandparent", "type": "PipelineReference" }, "parameters": {} } ], "type": "ScheduleTrigger", "typeProperties": { "recurrence": { "frequency": "Hour", "interval": 2, "startTime": "2020-04-06T15:00:00.000Z", "timeZone": "UTC" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/pipelines/01-Grandparent')]" ] } ] } ================================================ FILE: ARM Templates/Data Factory/v1.8.5 Export.json ================================================ { "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { "factoryName": { "type": "string", "metadata": "Data Factory name", "defaultValue": "" }, "Keys_properties_typeProperties_baseUrl": { "type": "string", "defaultValue": "" }, "FrameworkFunctions_properties_typeProperties_functionAppUrl": { "type": "string", "defaultValue": "" }, "SupportDatabase_properties_typeProperties_connectionString_secretName": { "type": "string", "defaultValue": "" } }, "variables": { "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" }, "resources": [ { "name": "[concat(parameters('factoryName'), '/GetSetMetadata')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" }, "annotations": [ "ADF.procfwk" ], "type": "AzureSqlTable", "schema": [], "typeProperties": {} }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/FunctionalTestingTrigger')]", "type": "Microsoft.DataFactory/factories/triggers", "apiVersion": "2018-06-01", "properties": { "description": "Used for functional testing of the framework in a dedicated environment.", "annotations": [ "ADF.procfwk" ], "runtimeState": "Stopped", "pipelines": [ { "pipelineReference": { "referenceName": "01-Grandparent", "type": "PipelineReference" }, "parameters": {} } ], "type": "ScheduleTrigger", "typeProperties": { "recurrence": { "frequency": "Hour", "interval": 2, "startTime": "2020-04-06T15:00:00Z", "timeZone": "UTC" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/pipelines/01-Grandparent')]" ] }, { "name": "[concat(parameters('factoryName'), '/Keys')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection to Key Vault for all other ADF linked service credentials required to run the processing framework.", "annotations": [ "ADF.procfwk" ], "type": "AzureKeyVault", "typeProperties": { "baseUrl": "[parameters('Keys_properties_typeProperties_baseUrl')]" } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/FrameworkFunctions')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [ "ADF.procfwk" ], "type": "AzureFunction", "typeProperties": { "functionAppUrl": "[parameters('FrameworkFunctions_properties_typeProperties_functionAppUrl')]", "functionKey": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "FrameworkFunctionsKey" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/SupportDatabase')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection between ADF and processing framework metadata SQLDB.", "annotations": [ "ADF.procfwk" ], "type": "AzureSqlDatabase", "typeProperties": { "connectionString": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "[parameters('SupportDatabase_properties_typeProperties_connectionString_secretName')]" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/04-Infant')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk infant pipeline used to check when the processing pipeline called by the Child completes and passes the resulting status back to the metadata database.", "activities": [ { "name": "Wait Until Pipeline Completes", "description": "Loops until the pipeline called completes.\n\nSimple status:\n- Running = new iteration.\n- Done = break.", "type": "Until", "dependsOn": [ { "activity": "Get Wait Duration", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@equals('Done',activity('Get Pipeline Status').output.SimpleStatus)", "type": "Expression" }, "activities": [ { "name": "Get Pipeline Status", "description": "Checks the status of a given processing pipeline and provides the value for the downstream framework activities to act upon.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "CheckPipelineStatus", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',pipeline().parameters.TenantId,'\",\n \"applicationId\": \"',pipeline().parameters.applicationId,'\",\n \"authenticationKey\": \"',pipeline().parameters.authenticationKey,'\",\n \"subscriptionId\": \"',pipeline().parameters.subscriptionId,'\",\n \"resourceGroup\": \"',pipeline().parameters.resourceGroup,'\",\n \"factoryName\": \"',pipeline().parameters.factoryName,'\",\n \"pipelineName\": \"',pipeline().parameters.pipelineName,'\",\n \"runId\": \"',pipeline().parameters.runId,'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Wait If Running", "description": "True = Do nothing.\nFalse = Wait.", "type": "IfCondition", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@equals('Done',activity('Get Pipeline Status').output.SimpleStatus)", "type": "Expression" }, "ifFalseActivities": [ { "name": "Wait for Pipeline", "description": "The processing pipeline is still running so Wait before checking its status again.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@activity('Get Wait Duration').output.firstRow.PropertyValue", "type": "Expression" } } } ] } }, { "name": "Set Last Check DateTime", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineLastStatusCheck]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Log Activity Failure", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "CallingActivity": { "value": "GetPipelineStatus", "type": "String" }, "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ], "timeout": "7.00:00:00" } }, { "name": "Set Pipeline Result", "description": "Receives the outcome from the function execution for a given processing pipeline and updates the current execution table with different pipelines status values depending on the result (case).", "type": "Switch", "dependsOn": [ { "activity": "Wait Until Pipeline Completes", "dependencyConditions": [ "Completed" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@activity('Get Pipeline Status').output.Status", "type": "Expression" }, "cases": [ { "value": "Succeeded", "activities": [ { "name": "Pipeline Status Succeeded", "description": "Updates the current execution table with a pipeline status of success if the function outcome is succeeded.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Failed", "activities": [ { "name": "Pipeline Status Failed", "description": "Updates the current execution table with a pipeline status of failed if the function outcome is failed. Also blocks pipelines in the downstream execution stage.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@activity('Get Pipeline Status').output.RunId", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Error Details", "description": "Get the activity error details for the run ID of the worker pipeline called. Returns an array of all errors.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "GetActivityErrors", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',pipeline().parameters.TenantId,'\",\n \"applicationId\": \"',pipeline().parameters.applicationId,'\",\n \"authenticationKey\": \"',pipeline().parameters.authenticationKey,'\",\n \"subscriptionId\": \"',pipeline().parameters.subscriptionId,'\",\n \"resourceGroup\": \"',pipeline().parameters.resourceGroup,'\",\n \"factoryName\": \"',pipeline().parameters.factoryName,'\",\n \"pipelineName\": \"',pipeline().parameters.pipelineName,'\",\n \"runId\": \"',pipeline().parameters.runId,'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Log Error Details", "description": "Parses pipeline error details and persists them to the metadata database error log table.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Error Details", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetErrorLogDetails]", "storedProcedureParameters": { "JsonErrorDetails": { "value": { "value": "@string(activity('Get Error Details').output)", "type": "Expression" }, "type": "String" }, "LocalExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Cancelled", "activities": [ { "name": "Pipeline Status Cancelled", "description": "Updates the current execution table with a pipeline status of unknown if the function returns an unexpected outcome.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineCancelled]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } ], "defaultActivities": [ { "name": "Pipeline Status Unknown", "description": "Updates the current execution table with a pipeline status of unknown if the function returns an unexpected outcome.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineUnknown]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Get Wait Duration", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "PipelineStatusCheckDuration" } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } } ], "parameters": { "tenantId": { "type": "string" }, "applicationId": { "type": "string" }, "authenticationKey": { "type": "string" }, "subscriptionId": { "type": "string" }, "resourceGroup": { "type": "string" }, "factoryName": { "type": "string" }, "pipelineName": { "type": "string" }, "runId": { "type": "string" }, "executionId": { "type": "string" }, "stageId": { "type": "int" }, "pipelineId": { "type": "int" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ], "lastPublishTime": "2020-08-17T11:35:19Z" }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/FrameworkFunctions')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/03-Child')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk child pipeline used to execute Worker pipelines within a given execution stage. This pipeline will be called once for each stage, then execute all Workers in parallel.", "activities": [ { "name": "Get Pipelines", "description": "Returns all pipelines from the metadata to be executed within a given processing stage.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelinesInStage]", "storedProcedureParameters": { "StageId": { "type": "Int32", "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Pipelines", "description": "Second level ForEach to run in parallel all pipelines within the stage. Items for iteration passed from the Get Pipelines lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Pipelines", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Pipelines').output.value", "type": "Expression" }, "isSequential": false, "batchCount": 40, "activities": [ { "name": "Execute Pipeline", "description": "The lowest level executor with the metadata framework to call existing processing pipelines within Data Factory. The function called will block processing and wait for an outcome.", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Log Pipeline Running", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Pipeline Params", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get SPN Details", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "ExecutePipeline", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n\t\"tenantId\": \"',pipeline().parameters.TenantId,'\",\n\t\"applicationId\": \"',activity('Get SPN Details').output.firstRow.Id,'\",\n\t\"authenticationKey\": \"',activity('Get SPN Details').output.firstRow.Secret,'\",\n\t\"subscriptionId\": \"',pipeline().parameters.SubscriptionId,'\",\n\t\"resourceGroup\": \"',item().ResourceGroupName,'\",\n\t\"factoryName\": \"',item().DataFactoryName,'\",\n\t\"pipelineName\": \"',item().PipelineName,'\"',activity('Get Pipeline Params').output.firstRow.Params,'\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Get Pipeline Params", "description": "Returns any parameters from metadata required for the processing pipeline being called. The output can be an empty string if no parameters are required.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelineParameters]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@item().PipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Pipeline Running", "description": "Sets the current pipeline with a status of running within the current execution database table.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get SPN Details", "description": "Return the SPN ID and Secret for the processing pipeline being executed. Called at this level as each pipeline can have a different SPN.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetServicePrincipal]", "storedProcedureParameters": { "DataFactory": { "type": "String", "value": { "value": "@item().DataFactoryName", "type": "Expression" } }, "PipelineName": { "type": "String", "value": { "value": "@item().PipelineName", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Activity Failure", "description": "Handle true failures from calling out to the Azure Function and update the current execution table accordingly so a restart can occur.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" }, "CallingActivity": { "value": "ExecutePipeline", "type": "String" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Running Pipeline Handler", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "04-Infant", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "tenantId": { "value": "@pipeline().parameters.TenantId", "type": "Expression" }, "applicationId": { "value": "@activity('Get SPN Details').output.firstRow.Id", "type": "Expression" }, "authenticationKey": { "value": "@activity('Get SPN Details').output.firstRow.Secret", "type": "Expression" }, "subscriptionId": { "value": "@pipeline().parameters.SubscriptionId", "type": "Expression" }, "resourceGroup": { "value": "@item().ResourceGroupName", "type": "Expression" }, "factoryName": { "value": "@item().DataFactoryName", "type": "Expression" }, "pipelineName": { "value": "@item().PipelineName", "type": "Expression" }, "runId": { "value": "@activity('Execute Pipeline').output.RunId", "type": "Expression" }, "executionId": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "stageId": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "pipelineId": { "value": "@item().PipelineId", "type": "Expression" } } } }, { "name": "Set Run Id", "description": "Provide the actual ADF run ID back to the current execution table for long term logging and alignment between the metadata other Azure monitoring tools.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunId]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@activity('Execute Pipeline').output.RunId", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check For Alerts", "description": "Checks the properties tables and if any recipients in the database require alerts sending for the current pipeline ID.", "type": "Lookup", "dependsOn": [ { "activity": "Set Run Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Running Pipeline Handler", "dependencyConditions": [ "Completed" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[CheckForEmailAlerts]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@item().PipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Send Alerts", "description": "True = alerts need sending.\nFalse = do nothing.", "type": "IfCondition", "dependsOn": [ { "activity": "Check For Alerts", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@activity('Check For Alerts').output.firstRow.SendAlerts", "type": "Expression" }, "ifTrueActivities": [ { "name": "Get Email Parts", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetEmailAlertParts]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@item().PipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Send Email", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Get Email Parts", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "SendEmail", "method": "POST", "headers": {}, "body": { "value": "@activity('Get Email Parts').output.firstRow", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } } ] } } ] } } ], "parameters": { "StageId": { "type": "int" }, "ExecutionId": { "type": "string" }, "TenantId": { "type": "string" }, "SubscriptionId": { "type": "string" } }, "variables": { "FunctionBody": { "type": "String" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ], "lastPublishTime": "2020-08-17T11:35:20Z" }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/FrameworkFunctions')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", "[concat(variables('factoryId'), '/pipelines/04-Infant')]" ] }, { "name": "[concat(parameters('factoryName'), '/02-Parent')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk parent pipeline used to bootstrap the orchestration framework in perform the first level ForEach calls in sequence for the metadata stages.", "activities": [ { "name": "Get Stages", "description": "Returns a distinct list of execution stages within the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Execution Wrapper", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetStages]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Stages", "description": "Top level ForEach to sequentially call all processing stages within the framework metadata. Items for iteration passed from the Get Stages lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Stages", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Tenant Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Subscription Id", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Stages').output.value", "type": "Expression" }, "isSequential": true, "activities": [ { "name": "Stage Executor", "description": "Call to the framework generic child pipeline for a given execution stage.", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Log Stage Preparing", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "03-Child", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "StageId": { "value": "@item().StageId", "type": "Expression" }, "ExecutionId": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" }, "TenantId": { "value": "@activity('Get Tenant Id').output.firstRow.PropertyValue", "type": "Expression" }, "SubscriptionId": { "value": "@activity('Get Subscription Id').output.firstRow.PropertyValue", "type": "Expression" } } } }, { "name": "Log Stage Preparing", "description": "Update the current execution table flagging all pipelines within the stage as preparing.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Check and Update Blockers", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogStagePreparing]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check and Update Blockers", "description": "Used to double check and stop the next execution stage if failures and blockers have be incurred. This also depends on the failure handling property value which defines the stored procedure behaviour.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[CheckForBlockedPipelines]", "storedProcedureParameters": { "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Execution Wrapper", "description": "Wrapper to reset and restart processing or create a completely new execution instance of the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Clean Up Previous Run", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[ExecutionWrapper]", "storedProcedureParameters": { "CallingDataFactory": { "type": "String", "value": { "value": "@pipeline().DataFactory", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Check Outcome and Update Logs", "description": "After a successful execution run the current execution metadata is moved to the long term logging table by this stored procedure call. Otherwise an error will be raised.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Stages", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[UpdateExecutionLog]", "storedProcedureParameters": { "PerformErrorCheck": { "value": { "value": "@bool(1)", "type": "Expression" }, "type": "Boolean" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Tenant Id", "description": "Returning the Azure Tenant Id from the metadata properties table.", "type": "Lookup", "dependsOn": [ { "activity": "Metadata Integrity Checks", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "TenantId" } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Get Subscription Id", "description": "Returning the Azure Subscription Id from the metadata properties table.", "type": "Lookup", "dependsOn": [ { "activity": "Metadata Integrity Checks", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "SubscriptionId" } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Metadata Integrity Checks", "description": "Performs a series of checks on all metadata held in the framework SQLDB. This is intended to raise errors before an execution run even starts.", "type": "Lookup", "dependsOn": [ { "activity": "Execute Precursor", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[CheckMetadataIntegrity]", "storedProcedureParameters": { "DebugMode": { "type": "Boolean", "value": "false" } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Clean Up Previous Run", "description": "Handle Worker pipelines that are reported as Running when the parent pipeline is called again. Get what the actual status of those pipelines is.", "type": "ForEach", "dependsOn": [ { "activity": "Metadata Integrity Checks", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Metadata Integrity Checks').output.value", "type": "Expression" }, "isSequential": false, "activities": [ { "name": "Get SPN Details", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetServicePrincipal]", "storedProcedureParameters": { "DataFactory": { "type": "String", "value": { "value": "@item().DataFactoryName", "type": "Expression" } }, "PipelineName": { "type": "String", "value": { "value": "@item().PipelineName", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Pipeline Checking", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineChecking]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Pipeline Status", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Get SPN Details", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Log Pipeline Checking", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "CheckPipelineStatus", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',item().TenantId,'\",\n \"applicationId\": \"',activity('Get SPN Details').output.firstRow.Id,'\",\n \"authenticationKey\": \"',activity('Get SPN Details').output.firstRow.Secret,'\",\n \"subscriptionId\": \"',item().SubscriptionId,'\",\n \"resourceGroup\": \"',item().ResourceGroupName,'\",\n \"factoryName\": \"',item().DataFactoryName,'\",\n \"pipelineName\": \"',item().PipelineName,'\",\n \"runId\": \"',item().AdfPipelineRunId,'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Set Pipeline Status", "type": "Switch", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@activity('Get Pipeline Status').output.Status", "type": "Expression" }, "cases": [ { "value": "Failed", "activities": [ { "name": "Pipeline Status Failed", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": null, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Succeeded", "activities": [ { "name": "Pipeline Status Succeeded", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Queued", "activities": [ { "name": "Pipeline Status Queued - Running", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "InProgress", "activities": [ { "name": "Pipeline Status InProgress - Running", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Cancelled", "activities": [ { "name": "Pipeline Status Cancelled", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineCancelled]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } ], "defaultActivities": [ { "name": "Pipeline Status Unknown", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineUnknown]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Set Last Check DateTime", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineLastStatusCheck]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Execute Precursor", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[ExecutePrecursorProcedure]" }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ], "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ], "lastPublishTime": "2020-08-17T11:35:21Z" }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", "[concat(variables('factoryId'), '/pipelines/03-Child')]", "[concat(variables('factoryId'), '/linkedServices/FrameworkFunctions')]" ] }, { "name": "[concat(parameters('factoryName'), '/01-Grandparent')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk grandparent pipeline used optionally to bootstrap any wider processes in your Data Factory that then calls the processing framework.", "activities": [ { "name": "Framework Processing", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "02-Parent", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": {} } } ], "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ], "lastPublishTime": "2020-08-17T11:35:22Z" }, "dependsOn": [ "[concat(variables('factoryId'), '/pipelines/02-Parent')]" ] } ] } ================================================ FILE: ARM Templates/Data Factory/v1.8.6 Export.json ================================================ { "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { "factoryName": { "type": "string", "metadata": "Data Factory name", "defaultValue": "" }, "FrameworkFunctions_properties_typeProperties_functionAppUrl": { "type": "string", "defaultValue": "" }, "Keys_properties_typeProperties_baseUrl": { "type": "string", "defaultValue": "" }, "SupportDatabase_properties_typeProperties_connectionString_secretName": { "type": "string", "defaultValue": "" } }, "variables": { "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" }, "resources": [ { "name": "[concat(parameters('factoryName'), '/01-Grandparent')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk grandparent pipeline used optionally to bootstrap any wider processes in your Data Factory that then calls the processing framework.", "activities": [ { "name": "Framework Processing", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "02-Parent", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": {} } } ], "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk", "Grandparent" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/pipelines/02-Parent')]" ] }, { "name": "[concat(parameters('factoryName'), '/02-Parent')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk parent pipeline used to bootstrap the orchestration framework in perform the first level ForEach calls in sequence for the metadata stages.", "activities": [ { "name": "Get Stages", "description": "Returns a distinct list of execution stages within the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Set Execution Id", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetStages]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@variables('ExecutionId')", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Stages", "description": "Top level ForEach to sequentially call all processing stages within the framework metadata. Items for iteration passed from the Get Stages lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Stages", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Tenant Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Subscription Id", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Stages').output.value", "type": "Expression" }, "isSequential": true, "activities": [ { "name": "Stage Executor", "description": "Call to the framework generic child pipeline for a given execution stage.", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Log Stage Preparing", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "03-Child", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "StageId": { "value": "@item().StageId", "type": "Expression" }, "ExecutionId": { "value": "@variables('ExecutionId')", "type": "Expression" }, "TenantId": { "value": "@activity('Get Tenant Id').output.firstRow.PropertyValue", "type": "Expression" }, "SubscriptionId": { "value": "@activity('Get Subscription Id').output.firstRow.PropertyValue", "type": "Expression" } } } }, { "name": "Log Stage Preparing", "description": "Update the current execution table flagging all pipelines within the stage as preparing.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Check and Update Blockers", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogStagePreparing]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@variables('ExecutionId')", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check and Update Blockers", "description": "Used to double check and stop the next execution stage if failures and blockers have be incurred. This also depends on the failure handling property value which defines the stored procedure behaviour.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[CheckForBlockedPipelines]", "storedProcedureParameters": { "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Execution Wrapper", "description": "Wrapper to reset and restart processing or create a completely new execution instance of the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Clean Up Previous Run", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[ExecutionWrapper]", "storedProcedureParameters": { "CallingDataFactory": { "type": "String", "value": { "value": "@pipeline().DataFactory", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Check Outcome and Update Logs", "description": "After a successful execution run the current execution metadata is moved to the long term logging table by this stored procedure call. Otherwise an error will be raised.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Stages", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[UpdateExecutionLog]", "storedProcedureParameters": { "PerformErrorCheck": { "value": { "value": "@bool(1)", "type": "Expression" }, "type": "Boolean" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Tenant Id", "description": "Returning the Azure Tenant Id from the metadata properties table.", "type": "Lookup", "dependsOn": [ { "activity": "Metadata Integrity Checks", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "TenantId" } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Get Subscription Id", "description": "Returning the Azure Subscription Id from the metadata properties table.", "type": "Lookup", "dependsOn": [ { "activity": "Metadata Integrity Checks", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "SubscriptionId" } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Metadata Integrity Checks", "description": "Performs a series of checks on all metadata held in the framework SQLDB. This is intended to raise errors before an execution run even starts.", "type": "Lookup", "dependsOn": [ { "activity": "Execute Precursor", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[CheckMetadataIntegrity]", "storedProcedureParameters": { "DebugMode": { "type": "Boolean", "value": "false" } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Clean Up Previous Run", "description": "Handle Worker pipelines that are reported as Running when the parent pipeline is called again. Get what the actual status of those pipelines is.", "type": "ForEach", "dependsOn": [ { "activity": "Metadata Integrity Checks", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Metadata Integrity Checks').output.value", "type": "Expression" }, "isSequential": false, "batchCount": 20, "activities": [ { "name": "Get SPN Details", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetServicePrincipal]", "storedProcedureParameters": { "DataFactory": { "type": "String", "value": { "value": "@item().DataFactoryName", "type": "Expression" } }, "PipelineName": { "type": "String", "value": { "value": "@item().PipelineName", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Pipeline Checking", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineChecking]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Pipeline Status", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Get SPN Details", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Log Pipeline Checking", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "CheckPipelineStatus", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',item().TenantId,'\",\n \"applicationId\": \"',activity('Get SPN Details').output.firstRow.Id,'\",\n \"authenticationKey\": \"',activity('Get SPN Details').output.firstRow.Secret,'\",\n \"subscriptionId\": \"',item().SubscriptionId,'\",\n \"resourceGroup\": \"',item().ResourceGroupName,'\",\n \"factoryName\": \"',item().DataFactoryName,'\",\n \"pipelineName\": \"',item().PipelineName,'\",\n \"runId\": \"',item().AdfPipelineRunId,'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Set Pipeline Status", "description": "Update the metadata depending on the actual pipeline outcome. Using the status as the case.", "type": "Switch", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@activity('Get Pipeline Status').output.Status", "type": "Expression" }, "cases": [ { "value": "Failed", "activities": [ { "name": "Pipeline Status Failed", "description": "Updates the current execution table with a pipeline status of failed if the function outcome is failed. Also blocks pipelines in the downstream execution stage.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": null, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Succeeded", "activities": [ { "name": "Pipeline Status Succeeded", "description": "Updates the current execution table with a pipeline status of success if the function outcome is succeeded.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Queued", "activities": [ { "name": "Pipeline Status Queued - Running", "description": "Updates the current execution table with a pipeline status of running if the function outcome is queued.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "InProgress", "activities": [ { "name": "Pipeline Status InProgress - Running", "description": "Updates the current execution table with a pipeline status of running if the function outcome is in progress.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Cancelled", "activities": [ { "name": "Pipeline Status Cancelled", "description": "Updates the current execution table with a pipeline status of cancelled if the function outcome is cancelled.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineCancelled]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } ], "defaultActivities": [ { "name": "Pipeline Status Unknown", "description": "Updates the current execution table with a pipeline status of unknown if the function returns an unexpected outcome.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineUnknown]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Set Last Check DateTime", "description": "Update the current execution table with a date time from when the function last checked the pipeline status.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineLastStatusCheck]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Execute Precursor", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[ExecutePrecursorProcedure]" }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Set Execution Id", "description": "Set the local execution Id to a pipeline variable for each in several downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Execution Wrapper", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "ExecutionId", "value": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" } } } ], "variables": { "ExecutionId": { "type": "String" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk", "Parent" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", "[concat(variables('factoryId'), '/pipelines/03-Child')]", "[concat(variables('factoryId'), '/linkedServices/FrameworkFunctions')]" ] }, { "name": "[concat(parameters('factoryName'), '/03-Child')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk child pipeline used to execute Worker pipelines within a given execution stage. This pipeline will be called once for each stage, then execute all Workers in parallel.", "activities": [ { "name": "Get Pipelines", "description": "Returns all pipelines from the metadata to be executed within a given processing stage.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelinesInStage]", "storedProcedureParameters": { "StageId": { "type": "Int32", "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Pipelines", "description": "Second level ForEach to run in parallel all pipelines within the stage. Items for iteration passed from the Get Pipelines lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Pipelines", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Pipelines').output.value", "type": "Expression" }, "isSequential": false, "batchCount": 50, "activities": [ { "name": "Execute Pipeline", "description": "The lowest level executor with the metadata framework to call existing processing pipelines within Data Factory. The function called will block processing and wait for an outcome.", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Log Pipeline Running", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Pipeline Params", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get SPN Details", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "ExecutePipeline", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n\t\"tenantId\": \"',pipeline().parameters.TenantId,'\",\n\t\"applicationId\": \"',activity('Get SPN Details').output.firstRow.Id,'\",\n\t\"authenticationKey\": \"',activity('Get SPN Details').output.firstRow.Secret,'\",\n\t\"subscriptionId\": \"',pipeline().parameters.SubscriptionId,'\",\n\t\"resourceGroup\": \"',item().ResourceGroupName,'\",\n\t\"factoryName\": \"',item().DataFactoryName,'\",\n\t\"pipelineName\": \"',item().PipelineName,'\"',activity('Get Pipeline Params').output.firstRow.Params,'\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Get Pipeline Params", "description": "Returns any parameters from metadata required for the processing pipeline being called. The output can be an empty string if no parameters are required.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelineParameters]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@item().PipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Pipeline Running", "description": "Sets the current pipeline with a status of running within the current execution database table.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get SPN Details", "description": "Return the SPN ID and Secret for the processing pipeline being executed. Called at this level as each pipeline can have a different SPN.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetServicePrincipal]", "storedProcedureParameters": { "DataFactory": { "type": "String", "value": { "value": "@item().DataFactoryName", "type": "Expression" } }, "PipelineName": { "type": "String", "value": { "value": "@item().PipelineName", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Activity Failure", "description": "Handle true failures from calling out to the Azure Function and update the current execution table accordingly so a restart can occur.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" }, "CallingActivity": { "value": "ExecutePipeline", "type": "String" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Running Pipeline Handler", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "04-Infant", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "tenantId": { "value": "@pipeline().parameters.TenantId", "type": "Expression" }, "applicationId": { "value": "@activity('Get SPN Details').output.firstRow.Id", "type": "Expression" }, "authenticationKey": { "value": "@activity('Get SPN Details').output.firstRow.Secret", "type": "Expression" }, "subscriptionId": { "value": "@pipeline().parameters.SubscriptionId", "type": "Expression" }, "resourceGroup": { "value": "@item().ResourceGroupName", "type": "Expression" }, "factoryName": { "value": "@item().DataFactoryName", "type": "Expression" }, "pipelineName": { "value": "@item().PipelineName", "type": "Expression" }, "runId": { "value": "@activity('Execute Pipeline').output.RunId", "type": "Expression" }, "executionId": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "stageId": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "pipelineId": { "value": "@item().PipelineId", "type": "Expression" } } } }, { "name": "Set Run Id", "description": "Provide the actual ADF run ID back to the current execution table for long term logging and alignment between the metadata other Azure monitoring tools.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunId]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@activity('Execute Pipeline').output.RunId", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check For Alerts", "description": "Checks the properties tables and if any recipients in the database require alerts sending for the current pipeline ID.", "type": "Lookup", "dependsOn": [ { "activity": "Set Run Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Running Pipeline Handler", "dependencyConditions": [ "Completed" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[CheckForEmailAlerts]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@item().PipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Send Alerts", "description": "True = alerts need sending.\nFalse = do nothing.", "type": "IfCondition", "dependsOn": [ { "activity": "Check For Alerts", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@activity('Check For Alerts').output.firstRow.SendAlerts", "type": "Expression" }, "ifTrueActivities": [ { "name": "Get Email Parts", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetEmailAlertParts]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@item().PipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Send Email", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Get Email Parts", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "SendEmail", "method": "POST", "headers": {}, "body": { "value": "@activity('Get Email Parts').output.firstRow", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } } ] } } ] } } ], "parameters": { "StageId": { "type": "int" }, "ExecutionId": { "type": "string" }, "TenantId": { "type": "string" }, "SubscriptionId": { "type": "string" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk", "Child" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/FrameworkFunctions')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", "[concat(variables('factoryId'), '/pipelines/04-Infant')]" ] }, { "name": "[concat(parameters('factoryName'), '/04-Infant')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk infant pipeline used to check when the processing pipeline called by the Child completes and passes the resulting status back to the metadata database.", "activities": [ { "name": "Wait Until Pipeline Completes", "description": "Loops until the Worker pipeline called completes.\n\nSimple status:\n- Running = new iteration.\n- Done = break.", "type": "Until", "dependsOn": [ { "activity": "Get Wait Duration", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@variables('WorkerPipelineState')", "type": "Expression" }, "activities": [ { "name": "Get Pipeline Status", "description": "Checks the status of a given processing pipeline and provides the value for the downstream framework activities to act upon.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "CheckPipelineStatus", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',pipeline().parameters.TenantId,'\",\n \"applicationId\": \"',pipeline().parameters.applicationId,'\",\n \"authenticationKey\": \"',pipeline().parameters.authenticationKey,'\",\n \"subscriptionId\": \"',pipeline().parameters.subscriptionId,'\",\n \"resourceGroup\": \"',pipeline().parameters.resourceGroup,'\",\n \"factoryName\": \"',pipeline().parameters.factoryName,'\",\n \"pipelineName\": \"',pipeline().parameters.pipelineName,'\",\n \"runId\": \"',pipeline().parameters.runId,'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Wait If Running", "description": "True = Do nothing.\nFalse = Wait, before the next iteration.", "type": "IfCondition", "dependsOn": [ { "activity": "Set Worker State", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@variables('WorkerPipelineState')", "type": "Expression" }, "ifFalseActivities": [ { "name": "Wait for Pipeline", "description": "The processing pipeline is still running so Wait before checking its status again.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@activity('Get Wait Duration').output.firstRow.PropertyValue", "type": "Expression" } } } ] } }, { "name": "Set Last Check DateTime", "description": "Update the current execution table with a date time from when the Worker pipeline status was last checked as part of the Until iterations.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineLastStatusCheck]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Log Activity Failure", "description": "Report to the current execution table that the framework pipeline activity has failed. This failure is outside of the scope of the framework and is probably related to a wider platform problem.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "CallingActivity": { "value": "GetPipelineStatus", "type": "String" }, "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Set Worker State", "description": "Set the bool state of the Worker pipeline to be used by the Until and If expressions. True = Complete, False = Running.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerPipelineState", "value": { "value": "@equals('Done',activity('Get Pipeline Status').output.SimpleStatus)", "type": "Expression" } } } ], "timeout": "7.00:00:00" } }, { "name": "Set Pipeline Result", "description": "Receives the outcome from the function execution for a given processing pipeline and updates the current execution table with different pipelines status values depending on the result (case).", "type": "Switch", "dependsOn": [ { "activity": "Wait Until Pipeline Completes", "dependencyConditions": [ "Completed" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@activity('Get Pipeline Status').output.Status", "type": "Expression" }, "cases": [ { "value": "Succeeded", "activities": [ { "name": "Pipeline Status Succeeded", "description": "Updates the current execution table with a pipeline status of success if the function outcome is succeeded.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Failed", "activities": [ { "name": "Pipeline Status Failed", "description": "Updates the current execution table with a pipeline status of failed if the function outcome is failed. Also blocks pipelines in the downstream execution stage.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@activity('Get Pipeline Status').output.RunId", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Error Details", "description": "Get the activity error details for the run ID of the worker pipeline called. Returns an array of all errors.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "GetActivityErrors", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',pipeline().parameters.TenantId,'\",\n \"applicationId\": \"',pipeline().parameters.applicationId,'\",\n \"authenticationKey\": \"',pipeline().parameters.authenticationKey,'\",\n \"subscriptionId\": \"',pipeline().parameters.subscriptionId,'\",\n \"resourceGroup\": \"',pipeline().parameters.resourceGroup,'\",\n \"factoryName\": \"',pipeline().parameters.factoryName,'\",\n \"pipelineName\": \"',pipeline().parameters.pipelineName,'\",\n \"runId\": \"',pipeline().parameters.runId,'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Log Error Details", "description": "Parses pipeline error details and persists them to the metadata database error log table.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Error Details", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetErrorLogDetails]", "storedProcedureParameters": { "JsonErrorDetails": { "value": { "value": "@string(activity('Get Error Details').output)", "type": "Expression" }, "type": "String" }, "LocalExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Cancelled", "activities": [ { "name": "Pipeline Status Cancelled", "description": "Updates the current execution table with a pipeline status of cancelled if the function outcome is cancelled.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineCancelled]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } ], "defaultActivities": [ { "name": "Pipeline Status Unknown", "description": "Updates the current execution table with a pipeline status of unknown if the function returns an unexpected outcome.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineUnknown]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Get Wait Duration", "description": "Return wait duration in seconds from database properties table to be used during each Until iteration when the Worker pipeline is still running.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "PipelineStatusCheckDuration" } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } } ], "parameters": { "tenantId": { "type": "string" }, "applicationId": { "type": "string" }, "authenticationKey": { "type": "string" }, "subscriptionId": { "type": "string" }, "resourceGroup": { "type": "string" }, "factoryName": { "type": "string" }, "pipelineName": { "type": "string" }, "runId": { "type": "string" }, "executionId": { "type": "string" }, "stageId": { "type": "int" }, "pipelineId": { "type": "int" } }, "variables": { "WorkerPipelineState": { "type": "Boolean" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk", "Infant" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/FrameworkFunctions')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/Intentional Error')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } }, { "name": "Raise Errors or Not", "type": "IfCondition", "dependsOn": [ { "activity": "Wait1", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@equals(pipeline().parameters.RaiseErrors,'true')", "type": "Expression" }, "ifTrueActivities": [ { "name": "Call Fail Procedure", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[dbo].[FailProcedure]", "storedProcedureParameters": { "RaiseError": { "value": { "value": "@pipeline().parameters.RaiseErrors", "type": "Expression" }, "type": "String" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } } ], "parameters": { "RaiseErrors": { "type": "string", "defaultValue": "false" }, "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/Wait 1')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 10')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait10", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 2')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait2", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 3')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait3", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 4')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait4", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 5')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait5", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 6')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait6", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 7')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait7", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 8')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait8", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 9')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait9", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 15 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/GetSetMetadata')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" }, "annotations": [ "ADF.procfwk" ], "type": "AzureSqlTable", "schema": [], "typeProperties": {} }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/FrameworkFunctions')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [ "ADF.procfwk" ], "type": "AzureFunction", "typeProperties": { "functionAppUrl": "[parameters('FrameworkFunctions_properties_typeProperties_functionAppUrl')]", "functionKey": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "FrameworkFunctionsKey" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/Keys')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection to Key Vault for all other ADF linked service credentials required to run the processing framework.", "annotations": [ "ADF.procfwk" ], "type": "AzureKeyVault", "typeProperties": { "baseUrl": "[parameters('Keys_properties_typeProperties_baseUrl')]" } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/SupportDatabase')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection between ADF and processing framework metadata SQLDB.", "annotations": [ "ADF.procfwk" ], "type": "AzureSqlDatabase", "typeProperties": { "connectionString": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "[parameters('SupportDatabase_properties_typeProperties_connectionString_secretName')]" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/FunctionalTestingTrigger')]", "type": "Microsoft.DataFactory/factories/triggers", "apiVersion": "2018-06-01", "properties": { "description": "Used for functional testing of the framework in a dedicated environment.", "annotations": [ "ADF.procfwk" ], "runtimeState": "Stopped", "pipelines": [ { "pipelineReference": { "referenceName": "01-Grandparent", "type": "PipelineReference" }, "parameters": {} } ], "type": "ScheduleTrigger", "typeProperties": { "recurrence": { "frequency": "Hour", "interval": 2, "startTime": "2020-04-06T15:00:00.000Z", "timeZone": "UTC" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/pipelines/01-Grandparent')]" ] } ] } ================================================ FILE: ARM Templates/Data Factory/v1.9 Export.json ================================================ { "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { "factoryName": { "type": "string", "metadata": "Data Factory name", "defaultValue": "" }, "FrameworkFunctions_properties_typeProperties_functionAppUrl": { "type": "string", "defaultValue": "" }, "Keys_properties_typeProperties_baseUrl": { "type": "string", "defaultValue": "" }, "SupportDatabase_properties_typeProperties_connectionString_secretName": { "type": "string", "defaultValue": "" } }, "variables": { "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" }, "resources": [ { "name": "[concat(parameters('factoryName'), '/01-Grandparent')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk grandparent pipeline used optionally to bootstrap any wider processes in your Data Factory that then calls the processing framework.", "activities": [ { "name": "Framework Processing", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "02-Parent", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": {} } } ], "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk", "Grandparent" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/pipelines/02-Parent')]" ] }, { "name": "[concat(parameters('factoryName'), '/02-Parent')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk parent pipeline used to bootstrap the orchestration framework in perform the first level ForEach calls in sequence for the metadata stages.", "activities": [ { "name": "Get Stages", "description": "Returns a distinct list of execution stages within the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Set Execution Id", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetStages]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@variables('ExecutionId')", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Stages", "description": "Top level ForEach to sequentially call all processing stages within the framework metadata. Items for iteration passed from the Get Stages lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Stages", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Stages').output.value", "type": "Expression" }, "isSequential": true, "activities": [ { "name": "Stage Executor", "description": "Call to the framework generic child pipeline for a given execution stage.", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Log Stage Preparing", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "03-Child", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "StageId": { "value": "@item().StageId", "type": "Expression" }, "ExecutionId": { "value": "@variables('ExecutionId')", "type": "Expression" } } } }, { "name": "Log Stage Preparing", "description": "Update the current execution table flagging all pipelines within the stage as preparing.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Check and Update Blockers", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogStagePreparing]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@variables('ExecutionId')", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check and Update Blockers", "description": "Used to double check and stop the next execution stage if failures and blockers have be incurred. This also depends on the failure handling property value which defines the stored procedure behaviour.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[CheckForBlockedPipelines]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@variables('ExecutionId')", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Execution Wrapper", "description": "Wrapper to reset and restart processing or create a completely new execution instance of the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Clean Up Previous Run", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[ExecutionWrapper]", "storedProcedureParameters": { "CallingDataFactory": { "type": "String", "value": { "value": "@pipeline().DataFactory", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Check Outcome and Update Logs", "description": "After a successful execution run the current execution metadata is moved to the long term logging table by this stored procedure call. Otherwise an error will be raised.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Stages", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[UpdateExecutionLog]", "storedProcedureParameters": { "PerformErrorCheck": { "value": { "value": "@bool(1)", "type": "Expression" }, "type": "Boolean" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Metadata Integrity Checks", "description": "Performs a series of checks on all metadata held in the framework SQLDB. This is intended to raise errors before an execution run even starts.", "type": "Lookup", "dependsOn": [ { "activity": "Execute Precursor", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[CheckMetadataIntegrity]", "storedProcedureParameters": { "DebugMode": { "type": "Boolean", "value": "false" } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Clean Up Previous Run", "description": "Handle Worker pipelines that are reported as Running when the parent pipeline is called again. Get what the actual status of those pipelines is.", "type": "ForEach", "dependsOn": [ { "activity": "Metadata Integrity Checks", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Metadata Integrity Checks').output.value", "type": "Expression" }, "isSequential": false, "batchCount": 20, "activities": [ { "name": "Get SPN Details", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetWorkerAuthDetails]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@item().LocalExecutionId", "type": "Expression" } }, "PipelineId": { "type": "Int32", "value": { "value": "@item().PipelineId", "type": "Expression" } }, "StageId": { "type": "Int32", "value": { "value": "@item().StageId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Pipeline Checking", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineChecking]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Pipeline Status", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Get SPN Details", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Log Pipeline Checking", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "CheckPipelineStatus", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',activity('Get SPN Details').output.firstRow.TenantId,'\",\n \"applicationId\": \"',activity('Get SPN Details').output.firstRow.AppId,'\",\n \"authenticationKey\": \"',activity('Get SPN Details').output.firstRow.AppSecret,'\",\n \"subscriptionId\": \"',activity('Get SPN Details').output.firstRow.SubscriptionId,'\",\n \"resourceGroup\": \"',item().ResourceGroupName,'\",\n \"factoryName\": \"',item().DataFactoryName,'\",\n \"pipelineName\": \"',item().PipelineName,'\",\n \"runId\": \"',item().AdfPipelineRunId,'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Set Pipeline Status", "description": "Update the metadata depending on the actual pipeline outcome. Using the status as the case.", "type": "Switch", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@activity('Get Pipeline Status').output.Status", "type": "Expression" }, "cases": [ { "value": "Failed", "activities": [ { "name": "Pipeline Status Failed", "description": "Updates the current execution table with a pipeline status of failed if the function outcome is failed. Also blocks pipelines in the downstream execution stage.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": null, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Succeeded", "activities": [ { "name": "Pipeline Status Succeeded", "description": "Updates the current execution table with a pipeline status of success if the function outcome is succeeded.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Queued", "activities": [ { "name": "Pipeline Status Queued - Running", "description": "Updates the current execution table with a pipeline status of running if the function outcome is queued.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "InProgress", "activities": [ { "name": "Pipeline Status InProgress - Running", "description": "Updates the current execution table with a pipeline status of running if the function outcome is in progress.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Cancelled", "activities": [ { "name": "Pipeline Status Cancelled", "description": "Updates the current execution table with a pipeline status of cancelled if the function outcome is cancelled.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineCancelled]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" }, "CleanUpRun": { "value": { "value": "@bool(1)", "type": "Expression" }, "type": "Boolean" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } ], "defaultActivities": [ { "name": "Pipeline Status Unknown", "description": "Updates the current execution table with a pipeline status of unknown if the function returns an unexpected outcome.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineUnknown]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" }, "CleanUpRun": { "value": { "value": "@bool(1)", "type": "Expression" }, "type": "Boolean" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Set Last Check DateTime", "description": "Update the current execution table with a date time from when the function last checked the pipeline status.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineLastStatusCheck]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Execute Precursor", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[ExecutePrecursorProcedure]" }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Set Execution Id", "description": "Set the local execution Id to a pipeline variable for each in several downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Execution Wrapper", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "ExecutionId", "value": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" } } } ], "variables": { "ExecutionId": { "type": "String" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk", "Parent" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", "[concat(variables('factoryId'), '/pipelines/03-Child')]", "[concat(variables('factoryId'), '/linkedServices/FrameworkFunctions')]" ] }, { "name": "[concat(parameters('factoryName'), '/03-Child')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk child pipeline used to execute Worker pipelines within a given execution stage. This pipeline will be called once for each stage, then execute all Workers in parallel.", "activities": [ { "name": "Get Pipelines", "description": "Returns all pipelines from the metadata to be executed within a given processing stage.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelinesInStage]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" } }, "StageId": { "type": "Int32", "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Pipelines", "description": "Second level ForEach to run in parallel all pipelines within the stage. Items for iteration passed from the Get Pipelines lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Pipelines", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Pipelines').output.value", "type": "Expression" }, "isSequential": false, "batchCount": 50, "activities": [ { "name": "Worker Pipeline Executor", "description": "Run the required worker pipeline and wait for its completion. Update metadata once done.", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "04-Infant", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "executionId": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "stageId": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "pipelineId": { "value": "@item().PipelineId", "type": "Expression" } } } } ] } } ], "parameters": { "StageId": { "type": "int" }, "ExecutionId": { "type": "string" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk", "Child" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/pipelines/04-Infant')]" ] }, { "name": "[concat(parameters('factoryName'), '/04-Infant')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk infant pipeline used to check when the processing pipeline called by the Child completes and passes the resulting status back to the metadata database.", "activities": [ { "name": "Execute Worker Pipeline", "description": "The lowest level executor with the metadata framework to call existing processing pipelines within Data Factory. The function called will block processing and wait for an outcome.", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Log Pipeline Running", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Pipeline Params", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set App Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set App Secret", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Subscription Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Tenant Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Pipeline Name", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Data Factory Name", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Resource Group", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "ExecutePipeline", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n\t\"tenantId\": \"',variables('WorkerTenantId'),'\",\n\t\"applicationId\": \"',variables('WorkerAppId'),'\",\n\t\"authenticationKey\": \"',variables('WorkerAppSecret'),'\",\n\t\"subscriptionId\": \"',variables('WorkerSubscriptionId'),'\",\n\t\"resourceGroup\": \"',variables('WorkerResourceGroup'),'\",\n\t\"factoryName\": \"',variables('WorkerDataFactoryName'),'\",\n\t\"pipelineName\": \"',variables('WorkerPipelineName'),'\"',activity('Get Pipeline Params').output.firstRow.Params,'\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Get Pipeline Params", "description": "Returns any parameters from metadata required for the processing pipeline being called. The output can be an empty string if no parameters are required.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelineParameters]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Pipeline Running", "description": "Sets the current pipeline with a status of running within the current execution database table.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Worker Authentication Details", "description": "Return the SPN ID and Secret for the worker pipeline being executed. Called at this level as each pipeline can have a different SPN.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetWorkerAuthDetails]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" } }, "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } }, "StageId": { "type": "Int32", "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Execute Function Activity Failure", "description": "Handle true failures from calling out to the Azure Function and update the current execution table accordingly so a restart can occur.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Worker Pipeline", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" }, "CallingActivity": { "value": "ExecuteWorkerPipeline", "type": "String" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Update Run Id", "description": "Provide the actual ADF run ID back to the current execution table for long term logging and alignment between the metadata other Azure monitoring tools.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Set Run Id", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunId]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@variables('WorkerRunId')", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check For Alerts", "description": "Checks the properties tables and if any recipients in the database require alerts sending for the current pipeline ID.", "type": "Lookup", "dependsOn": [ { "activity": "Update Run Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Pipeline Result", "dependencyConditions": [ "Completed" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[CheckForEmailAlerts]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Send Alerts", "description": "True = alerts need sending.\nFalse = do nothing.", "type": "IfCondition", "dependsOn": [ { "activity": "Check For Alerts", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@activity('Check For Alerts').output.firstRow.SendAlerts", "type": "Expression" }, "ifTrueActivities": [ { "name": "Get Email Parts", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetEmailAlertParts]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Send Email", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Get Email Parts", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "SendEmail", "method": "POST", "headers": {}, "body": { "value": "@activity('Get Email Parts').output.firstRow", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } } ] } }, { "name": "Wait Until Pipeline Completes", "description": "Loops until the Worker pipeline called completes.\n\nSimple status:\n- Running = new iteration.\n- Done = break.", "type": "Until", "dependsOn": [ { "activity": "Get Wait Duration", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Execute Worker Pipeline", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Run Id", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@variables('WorkerPipelineState')", "type": "Expression" }, "activities": [ { "name": "Get Worker Pipeline Status", "description": "Checks the status of a given processing pipeline and provides the value for the downstream framework activities to act upon.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "CheckPipelineStatus", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',variables('WorkerTenantId'),'\",\n \"applicationId\": \"',variables('WorkerAppId'),'\",\n \"authenticationKey\": \"',variables('WorkerAppSecret'),'\",\n \"subscriptionId\": \"',variables('WorkerSubscriptionId'),'\",\n \"resourceGroup\": \"',variables('WorkerResourceGroup'),'\",\n \"factoryName\": \"',variables('WorkerDataFactoryName'),'\",\n \"pipelineName\": \"',variables('WorkerPipelineName'),'\",\n \"runId\": \"',variables('WorkerRunId'),'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Wait If Running", "description": "True = Do nothing.\nFalse = Wait, before the next iteration.", "type": "IfCondition", "dependsOn": [ { "activity": "Set Worker State", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@variables('WorkerPipelineState')", "type": "Expression" }, "ifFalseActivities": [ { "name": "Wait for Pipeline", "description": "The processing pipeline is still running so Wait before checking its status again.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@activity('Get Wait Duration').output.firstRow.PropertyValue", "type": "Expression" } } } ] } }, { "name": "Set Last Check DateTime", "description": "Update the current execution table with a date time from when the Worker pipeline status was last checked as part of the Until iterations.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Worker Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineLastStatusCheck]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Log Check Function Activity Failure", "description": "Report to the current execution table that the framework pipeline activity has failed. This failure is outside of the scope of the framework and is probably related to a wider platform problem.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Worker Pipeline Status", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "CallingActivity": { "value": "GetWorkerPipelineStatus", "type": "String" }, "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Set Worker State", "description": "Set the bool state of the Worker pipeline to be used by the Until and If expressions. True = Complete, False = Running.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerPipelineState", "value": { "value": "@equals('Done',activity('Get Worker Pipeline Status').output.SimpleStatus)", "type": "Expression" } } } ], "timeout": "7.00:00:00" } }, { "name": "Set Pipeline Result", "description": "Receives the outcome from the function execution for a given processing pipeline and updates the current execution table with different pipelines status values depending on the result (case).", "type": "Switch", "dependsOn": [ { "activity": "Wait Until Pipeline Completes", "dependencyConditions": [ "Completed" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@activity('Get Worker Pipeline Status').output.Status", "type": "Expression" }, "cases": [ { "value": "Succeeded", "activities": [ { "name": "Pipeline Status Succeeded", "description": "Updates the current execution table with a pipeline status of success if the function outcome is succeeded.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Failed", "activities": [ { "name": "Pipeline Status Failed", "description": "Updates the current execution table with a pipeline status of failed if the function outcome is failed. Also blocks pipelines in the downstream execution stage.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@variables('WorkerRunId')", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Worker Pipeline Error Details", "description": "Get the activity error details for the run ID of the worker pipeline called. Returns an array of all errors.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "GetActivityErrors", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',variables('WorkerTenantId'),'\",\n \"applicationId\": \"',variables('WorkerAppId'),'\",\n \"authenticationKey\": \"',variables('WorkerAppSecret'),'\",\n \"subscriptionId\": \"',variables('WorkerSubscriptionId'),'\",\n \"resourceGroup\": \"',variables('WorkerResourceGroup'),'\",\n \"factoryName\": \"',variables('WorkerDataFactoryName'),'\",\n \"pipelineName\": \"',variables('WorkerPipelineName'),'\",\n \"runId\": \"',variables('WorkerRunId'),'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Log Error Details", "description": "Parses pipeline error details and persists them to the metadata database error log table.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Worker Pipeline Error Details", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetErrorLogDetails]", "storedProcedureParameters": { "JsonErrorDetails": { "value": { "value": "@string(activity('Get Worker Pipeline Error Details').output)", "type": "Expression" }, "type": "String" }, "LocalExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Cancelled", "activities": [ { "name": "Pipeline Status Cancelled", "description": "Updates the current execution table with a pipeline status of cancelled if the function outcome is cancelled.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineCancelled]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } ], "defaultActivities": [ { "name": "Pipeline Status Unknown", "description": "Updates the current execution table with a pipeline status of unknown if the function returns an unexpected outcome.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineUnknown]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Get Wait Duration", "description": "Return wait duration in seconds from database properties table to be used during each Until iteration when the Worker pipeline is still running.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "PipelineStatusCheckDuration" } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Set App Id", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Authentication Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerAppId", "value": { "value": "@activity('Get Worker Authentication Details').output.firstRow.AppId", "type": "Expression" } } }, { "name": "Set App Secret", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Authentication Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerAppSecret", "value": { "value": "@activity('Get Worker Authentication Details').output.firstRow.AppSecret", "type": "Expression" } } }, { "name": "Set Run Id", "type": "SetVariable", "dependsOn": [ { "activity": "Execute Worker Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerRunId", "value": { "value": "@activity('Execute Worker Pipeline').output.RunId", "type": "Expression" } } }, { "name": "Get Worker Pipeline Details", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetWorkerPipelineDetails]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" } }, "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } }, "StageId": { "type": "Int32", "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Set Tenant Id", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Authentication Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerTenantId", "value": { "value": "@activity('Get Worker Authentication Details').output.firstRow.TenantId", "type": "Expression" } } }, { "name": "Set Subscription Id", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Authentication Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerSubscriptionId", "value": { "value": "@activity('Get Worker Authentication Details').output.firstRow.SubscriptionId", "type": "Expression" } } }, { "name": "Set Pipeline Name", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Pipeline Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerPipelineName", "value": { "value": "@activity('Get Worker Pipeline Details').output.firstRow.PipelineName", "type": "Expression" } } }, { "name": "Set Data Factory Name", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Pipeline Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerDataFactoryName", "value": { "value": "@activity('Get Worker Pipeline Details').output.firstRow.DataFactoryName", "type": "Expression" } } }, { "name": "Set Resource Group", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Pipeline Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerResourceGroup", "value": { "value": "@activity('Get Worker Pipeline Details').output.firstRow.ResourceGroupName", "type": "Expression" } } } ], "parameters": { "executionId": { "type": "string" }, "stageId": { "type": "int" }, "pipelineId": { "type": "int" } }, "variables": { "WorkerPipelineState": { "type": "Boolean" }, "WorkerAppId": { "type": "String" }, "WorkerAppSecret": { "type": "String" }, "WorkerRunId": { "type": "String" }, "WorkerTenantId": { "type": "String" }, "WorkerSubscriptionId": { "type": "String" }, "WorkerPipelineName": { "type": "String" }, "WorkerDataFactoryName": { "type": "String" }, "WorkerResourceGroup": { "type": "String" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk", "Infant" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/FrameworkFunctions')]", "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/Intentional Error')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } }, { "name": "Raise Errors or Not", "type": "IfCondition", "dependsOn": [ { "activity": "Wait1", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@equals(pipeline().parameters.RaiseErrors,'true')", "type": "Expression" }, "ifTrueActivities": [ { "name": "Call Fail Procedure", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[dbo].[FailProcedure]", "storedProcedureParameters": { "RaiseError": { "value": { "value": "@pipeline().parameters.RaiseErrors", "type": "Expression" }, "type": "String" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } } ], "parameters": { "RaiseErrors": { "type": "string", "defaultValue": "false" }, "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/Wait 1')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 10')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait10", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 2')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait2", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 3')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait3", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 4')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait4", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 5')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait5", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 6')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait6", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 7')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait7", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 8')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait8", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 9')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait9", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 15 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/GetSetMetadata')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" }, "annotations": [ "ADF.procfwk" ], "type": "AzureSqlTable", "schema": [], "typeProperties": {} }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/FrameworkFunctions')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [ "ADF.procfwk" ], "type": "AzureFunction", "typeProperties": { "functionAppUrl": "[parameters('FrameworkFunctions_properties_typeProperties_functionAppUrl')]", "functionKey": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "FrameworkFunctionsKey" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/Keys')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection to Key Vault for all other ADF linked service credentials required to run the processing framework.", "annotations": [ "ADF.procfwk" ], "type": "AzureKeyVault", "typeProperties": { "baseUrl": "[parameters('Keys_properties_typeProperties_baseUrl')]" } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/SupportDatabase')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection between ADF and processing framework metadata SQLDB.", "annotations": [ "ADF.procfwk" ], "type": "AzureSqlDatabase", "typeProperties": { "connectionString": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "[parameters('SupportDatabase_properties_typeProperties_connectionString_secretName')]" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/FunctionalTestingTrigger')]", "type": "Microsoft.DataFactory/factories/triggers", "apiVersion": "2018-06-01", "properties": { "description": "Used for functional testing of the framework in a dedicated environment.", "annotations": [ "ADF.procfwk" ], "runtimeState": "Stopped", "pipelines": [ { "pipelineReference": { "referenceName": "01-Grandparent", "type": "PipelineReference" }, "parameters": {} } ], "type": "ScheduleTrigger", "typeProperties": { "recurrence": { "frequency": "Hour", "interval": 2, "startTime": "2020-04-06T15:00:00.000Z", "timeZone": "UTC" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/pipelines/01-Grandparent')]" ] } ] } ================================================ FILE: ARM Templates/Data Factory/v1.9.1 Export.json ================================================ { "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { "factoryName": { "type": "string", "metadata": "Data Factory name", "defaultValue": "" }, "FrameworkFunctions_properties_typeProperties_functionAppUrl": { "type": "string", "defaultValue": "" }, "Keys_properties_typeProperties_baseUrl": { "type": "string", "defaultValue": "" }, "SupportDatabase_properties_typeProperties_connectionString_secretName": { "type": "string", "defaultValue": "" } }, "variables": { "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" }, "resources": [ { "name": "[concat(parameters('factoryName'), '/01-Grandparent')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk grandparent pipeline used optionally to bootstrap any wider processes in your Data Factory that then calls the processing framework.", "activities": [ { "name": "Framework Processing", "description": "Call procfwk", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "02-Parent", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": {} } } ], "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk", "Grandparent" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/pipelines/02-Parent')]" ] }, { "name": "[concat(parameters('factoryName'), '/02-Parent')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk parent pipeline used to bootstrap the orchestration framework in perform the first level ForEach calls in sequence for the metadata stages.", "activities": [ { "name": "Get Stages", "description": "Returns a distinct list of execution stages within the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Set Execution Id", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetStages]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@variables('ExecutionId')", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Stages", "description": "Top level ForEach to sequentially call all processing stages within the framework metadata. Items for iteration passed from the Get Stages lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Stages", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Stages').output.value", "type": "Expression" }, "isSequential": true, "activities": [ { "name": "Stage Executor", "description": "Call to the framework generic child pipeline for a given execution stage.", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Log Stage Preparing", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "03-Child", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "StageId": { "value": "@item().StageId", "type": "Expression" }, "ExecutionId": { "value": "@variables('ExecutionId')", "type": "Expression" } } } }, { "name": "Log Stage Preparing", "description": "Update the current execution table flagging all pipelines within the stage as preparing.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Check and Update Blockers", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogStagePreparing]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@variables('ExecutionId')", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check and Update Blockers", "description": "Used to double check and stop the next execution stage if failures and blockers have be incurred. This also depends on the failure handling property value which defines the stored procedure behaviour.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[CheckForBlockedPipelines]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@variables('ExecutionId')", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Execution Wrapper", "description": "Wrapper to reset and restart processing or create a completely new execution instance of the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Clean Up Previous Run", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[ExecutionWrapper]", "storedProcedureParameters": { "CallingDataFactory": { "type": "String", "value": { "value": "@pipeline().DataFactory", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Check Outcome and Update Logs", "description": "After a successful execution run the current execution metadata is moved to the long term logging table by this stored procedure call. Otherwise an error will be raised.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Stages", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[UpdateExecutionLog]", "storedProcedureParameters": { "PerformErrorCheck": { "value": { "value": "@bool(1)", "type": "Expression" }, "type": "Boolean" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Metadata Integrity Checks", "description": "Performs a series of checks on all metadata held in the framework SQLDB. This is intended to raise errors before an execution run even starts.", "type": "Lookup", "dependsOn": [ { "activity": "Execute Precursor", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[CheckMetadataIntegrity]", "storedProcedureParameters": { "DebugMode": { "type": "Boolean", "value": "false" } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Clean Up Previous Run", "description": "Handle Worker pipelines that are reported as Running when the parent pipeline is called again. Get what the actual status of those pipelines is.", "type": "ForEach", "dependsOn": [ { "activity": "Metadata Integrity Checks", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Metadata Integrity Checks').output.value", "type": "Expression" }, "isSequential": false, "batchCount": 50, "activities": [ { "name": "Get SPN Details", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetWorkerAuthDetails]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@item().LocalExecutionId", "type": "Expression" } }, "PipelineId": { "type": "Int32", "value": { "value": "@item().PipelineId", "type": "Expression" } }, "StageId": { "type": "Int32", "value": { "value": "@item().StageId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Pipeline Checking", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineChecking]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Pipeline Status", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Get SPN Details", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Log Pipeline Checking", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "CheckPipelineStatus", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',activity('Get SPN Details').output.firstRow.TenantId,'\",\n \"applicationId\": \"',activity('Get SPN Details').output.firstRow.AppId,'\",\n \"authenticationKey\": \"',activity('Get SPN Details').output.firstRow.AppSecret,'\",\n \"subscriptionId\": \"',activity('Get SPN Details').output.firstRow.SubscriptionId,'\",\n \"resourceGroup\": \"',item().ResourceGroupName,'\",\n \"factoryName\": \"',item().DataFactoryName,'\",\n \"pipelineName\": \"',item().PipelineName,'\",\n \"runId\": \"',item().AdfPipelineRunId,'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Set Pipeline Status", "description": "Update the metadata depending on the actual pipeline outcome. Using the status as the case.", "type": "Switch", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@activity('Get Pipeline Status').output.Status", "type": "Expression" }, "cases": [ { "value": "Failed", "activities": [ { "name": "Pipeline Status Failed", "description": "Updates the current execution table with a pipeline status of failed if the function outcome is failed. Also blocks pipelines in the downstream execution stage.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": null, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Succeeded", "activities": [ { "name": "Pipeline Status Succeeded", "description": "Updates the current execution table with a pipeline status of success if the function outcome is succeeded.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Queued", "activities": [ { "name": "Pipeline Status Queued - Running", "description": "Updates the current execution table with a pipeline status of running if the function outcome is queued.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "InProgress", "activities": [ { "name": "Pipeline Status InProgress - Running", "description": "Updates the current execution table with a pipeline status of running if the function outcome is in progress.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Cancelled", "activities": [ { "name": "Pipeline Status Cancelled", "description": "Updates the current execution table with a pipeline status of cancelled if the function outcome is cancelled.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineCancelled]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" }, "CleanUpRun": { "value": { "value": "@bool(1)", "type": "Expression" }, "type": "Boolean" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } ], "defaultActivities": [ { "name": "Pipeline Status Unknown", "description": "Updates the current execution table with a pipeline status of unknown if the function returns an unexpected outcome.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineUnknown]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" }, "CleanUpRun": { "value": { "value": "@bool(1)", "type": "Expression" }, "type": "Boolean" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Set Last Check DateTime", "description": "Update the current execution table with a date time from when the function last checked the pipeline status.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineLastStatusCheck]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Execute Precursor", "description": "Uses the database property value ExecutionPrecursorProc to run any custom logic against the metadata database before the execution run starts.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[ExecutePrecursorProcedure]" }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Set Execution Id", "description": "Set the local execution Id to a pipeline variable for each in several downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Execution Wrapper", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "ExecutionId", "value": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" } } } ], "variables": { "ExecutionId": { "type": "String" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk", "Parent" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", "[concat(variables('factoryId'), '/pipelines/03-Child')]", "[concat(variables('factoryId'), '/linkedServices/FrameworkFunctions')]" ] }, { "name": "[concat(parameters('factoryName'), '/03-Child')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk child pipeline used to execute Worker pipelines within a given execution stage. This pipeline will be called once for each stage, then execute all Workers in parallel.", "activities": [ { "name": "Get Pipelines", "description": "Returns all pipelines from the metadata to be executed within a given processing stage.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelinesInStage]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" } }, "StageId": { "type": "Int32", "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Pipelines", "description": "Second level ForEach to run in parallel all pipelines within the stage. Items for iteration passed from the Get Pipelines lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Pipelines", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Pipelines').output.value", "type": "Expression" }, "isSequential": false, "batchCount": 50, "activities": [ { "name": "Worker Pipeline Executor", "description": "Run the required worker pipeline and wait for its completion. Update metadata once done.", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "04-Infant", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "executionId": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "stageId": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "pipelineId": { "value": "@item().PipelineId", "type": "Expression" } } } } ] } } ], "parameters": { "StageId": { "type": "int" }, "ExecutionId": { "type": "string" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk", "Child" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/pipelines/04-Infant')]" ] }, { "name": "[concat(parameters('factoryName'), '/04-Infant')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk infant pipeline used to check when the processing pipeline called by the Child completes and passes the resulting status back to the metadata database.", "activities": [ { "name": "Execute Worker Pipeline", "description": "The lowest level executor with the metadata framework to call existing processing pipelines within Data Factory. The function called will block processing and wait for an outcome.", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Log Pipeline Running", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Pipeline Params", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set App Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set App Secret", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Subscription Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Tenant Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Pipeline Name", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Data Factory Name", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Resource Group", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "ExecutePipeline", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n\t\"tenantId\": \"',variables('WorkerTenantId'),'\",\n\t\"applicationId\": \"',variables('WorkerAppId'),'\",\n\t\"authenticationKey\": \"',variables('WorkerAppSecret'),'\",\n\t\"subscriptionId\": \"',variables('WorkerSubscriptionId'),'\",\n\t\"resourceGroup\": \"',variables('WorkerResourceGroup'),'\",\n\t\"factoryName\": \"',variables('WorkerDataFactoryName'),'\",\n\t\"pipelineName\": \"',variables('WorkerPipelineName'),'\"',activity('Get Pipeline Params').output.firstRow.Params,'\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Get Pipeline Params", "description": "Returns any parameters from metadata required for the processing pipeline being called. The output can be an empty string if no parameters are required.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelineParameters]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Pipeline Running", "description": "Sets the current pipeline with a status of running within the current execution database table.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Worker Authentication Details", "description": "Return the SPN ID and Secret for the worker pipeline being executed. Called at this level as each pipeline can have a different SPN.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": true, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetWorkerAuthDetails]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" } }, "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } }, "StageId": { "type": "Int32", "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Execute Function Activity Failure", "description": "Handle true failures from calling out to the Azure Function and update the current execution table accordingly so a restart can occur.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Worker Pipeline", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" }, "CallingActivity": { "value": "ExecuteWorkerPipeline", "type": "String" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Update Run Id", "description": "Provide the actual ADF run ID back to the current execution table for long term logging and alignment between the metadata other Azure monitoring tools.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Set Run Id", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunId]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@variables('WorkerRunId')", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check For Alerts", "description": "Checks the properties tables and if any recipients in the database require alerts sending for the current pipeline ID.", "type": "Lookup", "dependsOn": [ { "activity": "Update Run Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Pipeline Result", "dependencyConditions": [ "Completed" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[CheckForEmailAlerts]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Send Alerts", "description": "True = alerts need sending.\nFalse = do nothing.", "type": "IfCondition", "dependsOn": [ { "activity": "Check For Alerts", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@activity('Check For Alerts').output.firstRow.SendAlerts", "type": "Expression" }, "ifTrueActivities": [ { "name": "Get Email Parts", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": true, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetEmailAlertParts]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Send Email", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Get Email Parts", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "SendEmail", "method": "POST", "headers": {}, "body": { "value": "@activity('Get Email Parts').output.firstRow", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } } ] } }, { "name": "Wait Until Pipeline Completes", "description": "Loops until the Worker pipeline called completes.\n\nSimple status:\n- Running = new iteration.\n- Done = break.", "type": "Until", "dependsOn": [ { "activity": "Get Wait Duration", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Execute Worker Pipeline", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Run Id", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@variables('WorkerPipelineState')", "type": "Expression" }, "activities": [ { "name": "Get Worker Pipeline Status", "description": "Checks the status of a given processing pipeline and provides the value for the downstream framework activities to act upon.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "CheckPipelineStatus", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',variables('WorkerTenantId'),'\",\n \"applicationId\": \"',variables('WorkerAppId'),'\",\n \"authenticationKey\": \"',variables('WorkerAppSecret'),'\",\n \"subscriptionId\": \"',variables('WorkerSubscriptionId'),'\",\n \"resourceGroup\": \"',variables('WorkerResourceGroup'),'\",\n \"factoryName\": \"',variables('WorkerDataFactoryName'),'\",\n \"pipelineName\": \"',variables('WorkerPipelineName'),'\",\n \"runId\": \"',variables('WorkerRunId'),'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Wait If Running", "description": "True = Do nothing.\nFalse = Wait, before the next iteration.", "type": "IfCondition", "dependsOn": [ { "activity": "Set Worker State", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@variables('WorkerPipelineState')", "type": "Expression" }, "ifFalseActivities": [ { "name": "Wait for Pipeline", "description": "The processing pipeline is still running so Wait before checking its status again.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@activity('Get Wait Duration').output.firstRow.PropertyValue", "type": "Expression" } } } ] } }, { "name": "Set Last Check DateTime", "description": "Update the current execution table with a date time from when the Worker pipeline status was last checked as part of the Until iterations.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Worker Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineLastStatusCheck]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Log Check Function Activity Failure", "description": "Report to the current execution table that the framework pipeline activity has failed. This failure is outside of the scope of the framework and is probably related to a wider platform problem.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Worker Pipeline Status", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "CallingActivity": { "value": "GetWorkerPipelineStatus", "type": "String" }, "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Set Worker State", "description": "Set the bool state of the Worker pipeline to be used by the Until and If expressions. True = Complete, False = Running.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerPipelineState", "value": { "value": "@equals('Done',activity('Get Worker Pipeline Status').output.SimpleStatus)", "type": "Expression" } } } ], "timeout": "0.00:10:00" } }, { "name": "Set Pipeline Result", "description": "Receives the outcome from the function execution for a given processing pipeline and updates the current execution table with different pipelines status values depending on the result (case).", "type": "Switch", "dependsOn": [ { "activity": "Wait Until Pipeline Completes", "dependencyConditions": [ "Completed" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@activity('Get Worker Pipeline Status').output.Status", "type": "Expression" }, "cases": [ { "value": "Succeeded", "activities": [ { "name": "Pipeline Status Succeeded", "description": "Updates the current execution table with a pipeline status of success if the function outcome is succeeded.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Failed", "activities": [ { "name": "Pipeline Status Failed", "description": "Updates the current execution table with a pipeline status of failed if the function outcome is failed. Also blocks pipelines in the downstream execution stage.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@variables('WorkerRunId')", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Worker Pipeline Error Details", "description": "Get the activity error details for the run ID of the worker pipeline called. Returns an array of all errors.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "GetActivityErrors", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',variables('WorkerTenantId'),'\",\n \"applicationId\": \"',variables('WorkerAppId'),'\",\n \"authenticationKey\": \"',variables('WorkerAppSecret'),'\",\n \"subscriptionId\": \"',variables('WorkerSubscriptionId'),'\",\n \"resourceGroup\": \"',variables('WorkerResourceGroup'),'\",\n \"factoryName\": \"',variables('WorkerDataFactoryName'),'\",\n \"pipelineName\": \"',variables('WorkerPipelineName'),'\",\n \"runId\": \"',variables('WorkerRunId'),'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Log Error Details", "description": "Parses pipeline error details and persists them to the metadata database error log table.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Worker Pipeline Error Details", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetErrorLogDetails]", "storedProcedureParameters": { "JsonErrorDetails": { "value": { "value": "@string(activity('Get Worker Pipeline Error Details').output)", "type": "Expression" }, "type": "String" }, "LocalExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Cancelled", "activities": [ { "name": "Pipeline Status Cancelled", "description": "Updates the current execution table with a pipeline status of cancelled if the function outcome is cancelled.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineCancelled]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } ], "defaultActivities": [ { "name": "Pipeline Status Unknown", "description": "Updates the current execution table with a pipeline status of unknown if the function returns an unexpected outcome.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineUnknown]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Get Wait Duration", "description": "Return wait duration in seconds from database properties table to be used during each Until iteration when the Worker pipeline is still running.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "PipelineStatusCheckDuration" } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Set App Id", "description": "Set local variable from activity output once for value reuse in downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Authentication Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerAppId", "value": { "value": "@activity('Get Worker Authentication Details').output.firstRow.AppId", "type": "Expression" } } }, { "name": "Set App Secret", "description": "Set local variable from activity output once for value reuse in downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Authentication Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerAppSecret", "value": { "value": "@activity('Get Worker Authentication Details').output.firstRow.AppSecret", "type": "Expression" } } }, { "name": "Set Run Id", "description": "Set local variable from activity output once for value reuse in downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Execute Worker Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerRunId", "value": { "value": "@activity('Execute Worker Pipeline').output.RunId", "type": "Expression" } } }, { "name": "Get Worker Pipeline Details", "description": "Return none sensitive worker pipeline information for metadata database. Including target data factory, pipeline name and resource group.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": true, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetWorkerPipelineDetails]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" } }, "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } }, "StageId": { "type": "Int32", "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Set Tenant Id", "description": "Set local variable from activity output once for value reuse in downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Authentication Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerTenantId", "value": { "value": "@activity('Get Worker Authentication Details').output.firstRow.TenantId", "type": "Expression" } } }, { "name": "Set Subscription Id", "description": "Set local variable from activity output once for value reuse in downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Authentication Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerSubscriptionId", "value": { "value": "@activity('Get Worker Authentication Details').output.firstRow.SubscriptionId", "type": "Expression" } } }, { "name": "Set Pipeline Name", "description": "Set local variable from activity output once for value reuse in downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Pipeline Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerPipelineName", "value": { "value": "@activity('Get Worker Pipeline Details').output.firstRow.PipelineName", "type": "Expression" } } }, { "name": "Set Data Factory Name", "description": "Set local variable from activity output once for value reuse in downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Pipeline Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerDataFactoryName", "value": { "value": "@activity('Get Worker Pipeline Details').output.firstRow.DataFactoryName", "type": "Expression" } } }, { "name": "Set Resource Group", "description": "Set local variable from activity output once for value reuse in downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Pipeline Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerResourceGroup", "value": { "value": "@activity('Get Worker Pipeline Details').output.firstRow.ResourceGroupName", "type": "Expression" } } } ], "parameters": { "executionId": { "type": "string" }, "stageId": { "type": "int" }, "pipelineId": { "type": "int" } }, "variables": { "WorkerPipelineState": { "type": "Boolean" }, "WorkerAppId": { "type": "String" }, "WorkerAppSecret": { "type": "String" }, "WorkerRunId": { "type": "String" }, "WorkerTenantId": { "type": "String" }, "WorkerSubscriptionId": { "type": "String" }, "WorkerPipelineName": { "type": "String" }, "WorkerDataFactoryName": { "type": "String" }, "WorkerResourceGroup": { "type": "String" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk", "Infant" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/FrameworkFunctions')]", "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/Intentional Error')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait1", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } }, { "name": "Raise Errors or Not", "description": "Framework development worker simulator.", "type": "IfCondition", "dependsOn": [ { "activity": "Wait1", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@equals(pipeline().parameters.RaiseErrors,'true')", "type": "Expression" }, "ifTrueActivities": [ { "name": "Call Fail Procedure", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[dbo].[FailProcedure]", "storedProcedureParameters": { "RaiseError": { "value": { "value": "@pipeline().parameters.RaiseErrors", "type": "Expression" }, "type": "String" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } } ], "parameters": { "RaiseErrors": { "type": "string", "defaultValue": "false" }, "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/Wait 1')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait1", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 10')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait10", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 2')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait2", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 3')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait3", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 4')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait4", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 5')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait5", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 6')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait6", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 7')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait7", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 8')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait8", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 9')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait9", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 15 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/GetSetMetadata')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "description": "Single generic dataset used to get and set all database metadata.", "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" }, "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ], "type": "AzureSqlTable", "schema": [], "typeProperties": {} }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/FrameworkFunctions')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Interact with the Azure Functions App used as middle ware when making requests to Worker pipelines. Authentication done at the Function App level.", "annotations": [ "ADF.procfwk" ], "type": "AzureFunction", "typeProperties": { "functionAppUrl": "[parameters('FrameworkFunctions_properties_typeProperties_functionAppUrl')]", "functionKey": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "FrameworkFunctionsKey" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/Keys')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection to Key Vault for all other ADF linked service credentials required to run the processing framework.", "annotations": [ "ADF.procfwk" ], "type": "AzureKeyVault", "typeProperties": { "baseUrl": "[parameters('Keys_properties_typeProperties_baseUrl')]" } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/SupportDatabase')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection between ADF and processing framework metadata SQLDB.", "annotations": [ "ADF.procfwk" ], "type": "AzureSqlDatabase", "typeProperties": { "connectionString": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "[parameters('SupportDatabase_properties_typeProperties_connectionString_secretName')]" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/FunctionalTestingTrigger')]", "type": "Microsoft.DataFactory/factories/triggers", "apiVersion": "2018-06-01", "properties": { "description": "Used for functional testing of the framework in a dedicated environment.", "annotations": [ "ADF.procfwk" ], "runtimeState": "Stopped", "pipelines": [ { "pipelineReference": { "referenceName": "01-Grandparent", "type": "PipelineReference" }, "parameters": {} } ], "type": "ScheduleTrigger", "typeProperties": { "recurrence": { "frequency": "Hour", "interval": 2, "startTime": "2020-04-06T15:00:00Z", "timeZone": "UTC" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/pipelines/01-Grandparent')]" ] } ] } ================================================ FILE: ARM Templates/Data Factory/v1.9.2 Export.json ================================================ { "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { "factoryName": { "type": "string", "metadata": "Data Factory name", "defaultValue": "" }, "FrameworkFunctions_properties_typeProperties_functionAppUrl": { "type": "string", "defaultValue": "" }, "Keys_properties_typeProperties_baseUrl": { "type": "string", "defaultValue": "" }, "SupportDatabase_properties_typeProperties_connectionString_secretName": { "type": "string", "defaultValue": "" } }, "variables": { "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" }, "resources": [ { "name": "[concat(parameters('factoryName'), '/01-Grandparent')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk grandparent pipeline used optionally to bootstrap any wider processes in your Data Factory that then calls the processing framework.", "activities": [ { "name": "procfwk", "description": "Call procfwk", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "02-Parent", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "BatchName": { "value": "@pipeline().parameters.BatchName", "type": "Expression" } } } } ], "parameters": { "BatchName": { "type": "string", "defaultValue": "NotUsed" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "procfwk", "Grandparent" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/pipelines/02-Parent')]" ] }, { "name": "[concat(parameters('factoryName'), '/02-Parent')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk parent pipeline used to bootstrap the orchestration framework in perform the first level ForEach calls in sequence for the metadata stages.", "activities": [ { "name": "Get Stages", "description": "Returns a distinct list of execution stages within the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Set Execution Id", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetStages]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@variables('ExecutionId')", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Stages", "description": "Top level ForEach to sequentially call all processing stages within the framework metadata. Items for iteration passed from the Get Stages lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Stages", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Stages').output.value", "type": "Expression" }, "isSequential": true, "activities": [ { "name": "Stage Executor", "description": "Call to the framework generic child pipeline for a given execution stage.", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Log Stage Preparing", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "03-Child", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "StageId": { "value": "@item().StageId", "type": "Expression" }, "ExecutionId": { "value": "@variables('ExecutionId')", "type": "Expression" } } } }, { "name": "Log Stage Preparing", "description": "Update the current execution table flagging all pipelines within the stage as preparing.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Check and Update Blockers", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogStagePreparing]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@variables('ExecutionId')", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check and Update Blockers", "description": "Used to double check and stop the next execution stage if failures and blockers have be incurred. This also depends on the failure handling property value which defines the stored procedure behaviour.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[CheckForBlockedPipelines]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@variables('ExecutionId')", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Execution Wrapper", "description": "Wrapper to reset and restart processing or create a completely new execution instance of the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Clean Up Previous Run", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[ExecutionWrapper]", "storedProcedureParameters": { "CallingDataFactory": { "type": "String", "value": { "value": "@pipeline().DataFactory", "type": "Expression" } }, "BatchName": { "type": "String", "value": { "value": "@pipeline().parameters.BatchName", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Check Outcome and Update Logs", "description": "After a successful execution run the current execution metadata is moved to the long term logging table by this stored procedure call. Otherwise an error will be raised.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Stages", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[UpdateExecutionLog]", "storedProcedureParameters": { "PerformErrorCheck": { "value": { "value": "@bool(1)", "type": "Expression" }, "type": "Boolean" }, "ExecutionId": { "value": { "value": "@variables('ExecutionId')", "type": "Expression" }, "type": "Guid" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check Previous Execution", "description": "Query the current execution table for worker pipelines that require a clean up from the previous execution run.", "type": "Lookup", "dependsOn": [ { "activity": "Execute Precursor", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[CheckPreviousExeuction]", "storedProcedureParameters": { "BatchName": { "type": "String", "value": { "value": "@pipeline().parameters.BatchName", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Clean Up Previous Run", "description": "Handle Worker pipelines that are reported as Running when the parent pipeline is called again. Get what the actual status of those pipelines is.", "type": "ForEach", "dependsOn": [ { "activity": "Check Previous Execution", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Check Metadata Integrity", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Check Previous Execution').output.value", "type": "Expression" }, "isSequential": false, "batchCount": 50, "activities": [ { "name": "Get SPN Details", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": true, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetWorkerAuthDetails]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@item().LocalExecutionId", "type": "Expression" } }, "PipelineId": { "type": "Int32", "value": { "value": "@item().PipelineId", "type": "Expression" } }, "StageId": { "type": "Int32", "value": { "value": "@item().StageId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Pipeline Checking", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineChecking]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Pipeline Status", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Get SPN Details", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Log Pipeline Checking", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "CheckPipelineStatus", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',activity('Get SPN Details').output.firstRow.TenantId,'\",\n \"applicationId\": \"',activity('Get SPN Details').output.firstRow.AppId,'\",\n \"authenticationKey\": \"',activity('Get SPN Details').output.firstRow.AppSecret,'\",\n \"subscriptionId\": \"',activity('Get SPN Details').output.firstRow.SubscriptionId,'\",\n \"resourceGroup\": \"',item().ResourceGroupName,'\",\n \"factoryName\": \"',item().DataFactoryName,'\",\n \"pipelineName\": \"',item().PipelineName,'\",\n \"runId\": \"',item().AdfPipelineRunId,'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Set Pipeline Status", "description": "Update the metadata depending on the actual pipeline outcome. Using the status as the case.", "type": "Switch", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@activity('Get Pipeline Status').output.Status", "type": "Expression" }, "cases": [ { "value": "Failed", "activities": [ { "name": "Pipeline Status Failed", "description": "Updates the current execution table with a pipeline status of failed if the function outcome is failed. Also blocks pipelines in the downstream execution stage.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": null, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Succeeded", "activities": [ { "name": "Pipeline Status Succeeded", "description": "Updates the current execution table with a pipeline status of success if the function outcome is succeeded.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Queued", "activities": [ { "name": "Pipeline Status Queued - Running", "description": "Updates the current execution table with a pipeline status of running if the function outcome is queued.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "InProgress", "activities": [ { "name": "Pipeline Status InProgress - Running", "description": "Updates the current execution table with a pipeline status of running if the function outcome is in progress.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Cancelled", "activities": [ { "name": "Pipeline Status Cancelled", "description": "Updates the current execution table with a pipeline status of cancelled if the function outcome is cancelled.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineCancelled]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" }, "CleanUpRun": { "value": { "value": "@bool(1)", "type": "Expression" }, "type": "Boolean" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } ], "defaultActivities": [ { "name": "Pipeline Status Unknown", "description": "Updates the current execution table with a pipeline status of unknown if the function returns an unexpected outcome.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineUnknown]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" }, "CleanUpRun": { "value": { "value": "@bool(1)", "type": "Expression" }, "type": "Boolean" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Set Last Check DateTime", "description": "Update the current execution table with a date time from when the function last checked the pipeline status.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineLastStatusCheck]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Execute Precursor", "description": "Uses the database property value ExecutionPrecursorProc to run any custom logic against the metadata database before the execution run starts.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Is Parent Already Running", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[ExecutePrecursorProcedure]" }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Set Execution Id", "description": "Set the local execution Id to a pipeline variable for each in several downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Execution Wrapper", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "ExecutionId", "value": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" } } }, { "name": "Is Parent Already Running", "description": "Establish before anything else if the parent pipeline is already running. Batch execution aware.", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "Check For Running Pipeline", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "BatchName": { "value": "@pipeline().parameters.BatchName", "type": "Expression" }, "PipelineName": { "value": "@pipeline().Pipeline", "type": "Expression" }, "ThisRunId": { "value": "@pipeline().RunId", "type": "Expression" } } } }, { "name": "Check Metadata Integrity", "description": "Performs a series of checks on all metadata held in the framework SQLDB. This is intended to raise errors before an execution run even starts.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Precursor", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[CheckMetadataIntegrity]", "storedProcedureParameters": { "BatchName": { "value": { "value": "@pipeline().parameters.BatchName", "type": "Expression" }, "type": "String" }, "DebugMode": { "value": { "value": "@bool(0)", "type": "Expression" }, "type": "Boolean" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ], "parameters": { "BatchName": { "type": "string", "defaultValue": "NotUsed" } }, "variables": { "ExecutionId": { "type": "String" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "procfwk", "Parent" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", "[concat(variables('factoryId'), '/pipelines/Check For Running Pipeline')]", "[concat(variables('factoryId'), '/pipelines/03-Child')]", "[concat(variables('factoryId'), '/linkedServices/FrameworkFunctions')]" ] }, { "name": "[concat(parameters('factoryName'), '/03-Child')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk child pipeline used to execute Worker pipelines within a given execution stage. This pipeline will be called once for each stage, then execute all Workers in parallel.", "activities": [ { "name": "Get Pipelines", "description": "Returns all pipelines from the metadata to be executed within a given processing stage.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelinesInStage]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" } }, "StageId": { "type": "Int32", "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Pipelines", "description": "Second level ForEach to run in parallel all pipelines within the stage. Items for iteration passed from the Get Pipelines lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Pipelines", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Pipelines').output.value", "type": "Expression" }, "isSequential": false, "batchCount": 50, "activities": [ { "name": "Worker Pipeline Executor", "description": "Run the required worker pipeline and wait for its completion. Update metadata once done.", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "04-Infant", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "executionId": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "stageId": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "pipelineId": { "value": "@item().PipelineId", "type": "Expression" } } } } ] } } ], "parameters": { "StageId": { "type": "int" }, "ExecutionId": { "type": "string" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "procfwk", "Child" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/pipelines/04-Infant')]" ] }, { "name": "[concat(parameters('factoryName'), '/04-Infant')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk infant pipeline used to check when the processing pipeline called by the Child completes and passes the resulting status back to the metadata database.", "activities": [ { "name": "Execute Worker Pipeline", "description": "The lowest level executor with the metadata framework to call existing processing pipelines within Data Factory. The function called will block processing and wait for an outcome.", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Log Pipeline Running", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Pipeline Params", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "ExecutePipeline", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n\t\"tenantId\": \"',variables('WorkerTenantId'),'\",\n\t\"applicationId\": \"',variables('WorkerAppId'),'\",\n\t\"authenticationKey\": \"',variables('WorkerAppSecret'),'\",\n\t\"subscriptionId\": \"',variables('WorkerSubscriptionId'),'\",\n\t\"resourceGroup\": \"',variables('WorkerResourceGroup'),'\",\n\t\"factoryName\": \"',variables('WorkerDataFactoryName'),'\",\n\t\"pipelineName\": \"',variables('WorkerPipelineName'),'\"',activity('Get Pipeline Params').output.firstRow.Params,'\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Get Pipeline Params", "description": "Returns any parameters from metadata required for the processing pipeline being called. The output can be an empty string if no parameters are required.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelineParameters]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Pipeline Running", "description": "Sets the current pipeline with a status of running within the current execution database table.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Is Target Worker Validate", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Worker Authentication Details", "description": "Return the SPN ID and Secret for the worker pipeline being executed. Called at this level as each pipeline can have a different SPN.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": true, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetWorkerAuthDetails]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" } }, "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } }, "StageId": { "type": "Int32", "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Execute Function Activity Failure", "description": "Handle true failures from calling out to the Azure Function and update the current execution table accordingly so a restart can occur.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Worker Pipeline", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" }, "CallingActivity": { "value": "ExecuteWorkerPipeline", "type": "String" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Update Run Id", "description": "Provide the actual ADF run ID back to the current execution table for long term logging and alignment between the metadata other Azure monitoring tools.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Set Run Id", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunId]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@variables('WorkerRunId')", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check For Alerts", "description": "Checks the properties tables and if any recipients in the database require alerts sending for the current pipeline ID.", "type": "Lookup", "dependsOn": [ { "activity": "Update Run Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Pipeline Result", "dependencyConditions": [ "Completed" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[CheckForEmailAlerts]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Send Alerts", "description": "True = alerts need sending.\nFalse = do nothing.", "type": "IfCondition", "dependsOn": [ { "activity": "Check For Alerts", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@activity('Check For Alerts').output.firstRow.SendAlerts", "type": "Expression" }, "ifTrueActivities": [ { "name": "Get Email Parts", "description": "Return all required content from the metadata database to send an email alerting using the procfwk. The lookup returns the exact content for the function body request.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": true, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetEmailAlertParts]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Send Email", "description": "Use an Azure Function to perform an SMTP client email send operation.", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Get Email Parts", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "SendEmail", "method": "POST", "headers": {}, "body": { "value": "@activity('Get Email Parts').output.firstRow", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } } ] } }, { "name": "Wait Until Pipeline Completes", "description": "Loops until the Worker pipeline called completes.\n\nSimple status:\n- Running = new iteration.\n- Done = break.", "type": "Until", "dependsOn": [ { "activity": "Get Wait Duration", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Execute Worker Pipeline", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Run Id", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@variables('WorkerPipelineState')", "type": "Expression" }, "activities": [ { "name": "Get Worker Pipeline Status", "description": "Checks the status of a given processing pipeline and provides the value for the downstream framework activities to act upon.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "CheckPipelineStatus", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',variables('WorkerTenantId'),'\",\n \"applicationId\": \"',variables('WorkerAppId'),'\",\n \"authenticationKey\": \"',variables('WorkerAppSecret'),'\",\n \"subscriptionId\": \"',variables('WorkerSubscriptionId'),'\",\n \"resourceGroup\": \"',variables('WorkerResourceGroup'),'\",\n \"factoryName\": \"',variables('WorkerDataFactoryName'),'\",\n \"pipelineName\": \"',variables('WorkerPipelineName'),'\",\n \"runId\": \"',variables('WorkerRunId'),'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Wait If Running", "description": "True = Do nothing.\nFalse = Wait, before the next iteration.", "type": "IfCondition", "dependsOn": [ { "activity": "Set Worker State", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@variables('WorkerPipelineState')", "type": "Expression" }, "ifFalseActivities": [ { "name": "Wait for Pipeline", "description": "The processing pipeline is still running so Wait before checking its status again.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@activity('Get Wait Duration').output.firstRow.PropertyValue", "type": "Expression" } } } ] } }, { "name": "Set Last Check DateTime", "description": "Update the current execution table with a date time from when the Worker pipeline status was last checked as part of the Until iterations.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Worker Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineLastStatusCheck]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Log Check Function Activity Failure", "description": "Report to the current execution table that the framework pipeline activity has failed. This failure is outside of the scope of the framework and is probably related to a wider platform problem.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Worker Pipeline Status", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "CallingActivity": { "value": "GetWorkerPipelineStatus", "type": "String" }, "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Set Worker State", "description": "Set the bool state of the Worker pipeline to be used by the Until and If expressions. True = Complete, False = Running.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerPipelineState", "value": { "value": "@equals('Done',activity('Get Worker Pipeline Status').output.SimpleStatus)", "type": "Expression" } } } ], "timeout": "0.00:10:00" } }, { "name": "Set Pipeline Result", "description": "Receives the outcome from the function execution for a given processing pipeline and updates the current execution table with different pipelines status values depending on the result (case).", "type": "Switch", "dependsOn": [ { "activity": "Wait Until Pipeline Completes", "dependencyConditions": [ "Completed" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@activity('Get Worker Pipeline Status').output.Status", "type": "Expression" }, "cases": [ { "value": "Succeeded", "activities": [ { "name": "Pipeline Status Succeeded", "description": "Updates the current execution table with a pipeline status of success if the function outcome is succeeded.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Failed", "activities": [ { "name": "Pipeline Status Failed", "description": "Updates the current execution table with a pipeline status of failed if the function outcome is failed. Also blocks pipelines in the downstream execution stage.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@variables('WorkerRunId')", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Worker Pipeline Error Details", "description": "Get the activity error details for the run ID of the worker pipeline called. Returns an array of all errors.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "GetActivityErrors", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',variables('WorkerTenantId'),'\",\n \"applicationId\": \"',variables('WorkerAppId'),'\",\n \"authenticationKey\": \"',variables('WorkerAppSecret'),'\",\n \"subscriptionId\": \"',variables('WorkerSubscriptionId'),'\",\n \"resourceGroup\": \"',variables('WorkerResourceGroup'),'\",\n \"factoryName\": \"',variables('WorkerDataFactoryName'),'\",\n \"pipelineName\": \"',variables('WorkerPipelineName'),'\",\n \"runId\": \"',variables('WorkerRunId'),'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Log Error Details", "description": "Parses pipeline error details and persists them to the metadata database error log table.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Worker Pipeline Error Details", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetErrorLogDetails]", "storedProcedureParameters": { "JsonErrorDetails": { "value": { "value": "@string(activity('Get Worker Pipeline Error Details').output)", "type": "Expression" }, "type": "String" }, "LocalExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Cancelled", "activities": [ { "name": "Pipeline Status Cancelled", "description": "Updates the current execution table with a pipeline status of cancelled if the function outcome is cancelled.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineCancelled]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } ], "defaultActivities": [ { "name": "Pipeline Status Unknown", "description": "Updates the current execution table with a pipeline status of unknown if the function returns an unexpected outcome.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineUnknown]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Get Wait Duration", "description": "Return wait duration in seconds from database properties table to be used during each Until iteration when the Worker pipeline is still running.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "PipelineStatusCheckDuration" } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Set App Id", "description": "Set local variable from activity output once for value reuse in downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Authentication Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerAppId", "value": { "value": "@activity('Get Worker Authentication Details').output.firstRow.AppId", "type": "Expression" } } }, { "name": "Set App Secret", "description": "Set local variable from activity output once for value reuse in downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Authentication Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerAppSecret", "value": { "value": "@activity('Get Worker Authentication Details').output.firstRow.AppSecret", "type": "Expression" } } }, { "name": "Set Run Id", "description": "Set local variable from activity output once for value reuse in downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Execute Worker Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerRunId", "value": { "value": "@activity('Execute Worker Pipeline').output.RunId", "type": "Expression" } } }, { "name": "Get Worker Pipeline Details", "description": "Return none sensitive worker pipeline information for metadata database. Including target data factory, pipeline name and resource group.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": true, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetWorkerPipelineDetails]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" } }, "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } }, "StageId": { "type": "Int32", "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Set Tenant Id", "description": "Set local variable from activity output once for value reuse in downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Authentication Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerTenantId", "value": { "value": "@activity('Get Worker Authentication Details').output.firstRow.TenantId", "type": "Expression" } } }, { "name": "Set Subscription Id", "description": "Set local variable from activity output once for value reuse in downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Authentication Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerSubscriptionId", "value": { "value": "@activity('Get Worker Authentication Details').output.firstRow.SubscriptionId", "type": "Expression" } } }, { "name": "Set Pipeline Name", "description": "Set local variable from activity output once for value reuse in downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Pipeline Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerPipelineName", "value": { "value": "@activity('Get Worker Pipeline Details').output.firstRow.PipelineName", "type": "Expression" } } }, { "name": "Set Data Factory Name", "description": "Set local variable from activity output once for value reuse in downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Pipeline Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerDataFactoryName", "value": { "value": "@activity('Get Worker Pipeline Details').output.firstRow.DataFactoryName", "type": "Expression" } } }, { "name": "Set Resource Group", "description": "Set local variable from activity output once for value reuse in downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Pipeline Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerResourceGroup", "value": { "value": "@activity('Get Worker Pipeline Details').output.firstRow.ResourceGroupName", "type": "Expression" } } }, { "name": "Validate Pipeline", "description": "Query the target data factory and establish if the provided worker pipeline name is valid.", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Set Resource Group", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Data Factory Name", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Subscription Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set App Secret", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set App Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Pipeline Name", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Tenant Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Log Pipeline Validating", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "ValidatePipeline", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n\t\"tenantId\": \"',variables('WorkerTenantId'),'\",\n\t\"applicationId\": \"',variables('WorkerAppId'),'\",\n\t\"authenticationKey\": \"',variables('WorkerAppSecret'),'\",\n\t\"subscriptionId\": \"',variables('WorkerSubscriptionId'),'\",\n\t\"resourceGroup\": \"',variables('WorkerResourceGroup'),'\",\n\t\"factoryName\": \"',variables('WorkerDataFactoryName'),'\",\n\t\"pipelineName\": \"',variables('WorkerPipelineName'),'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Is Target Worker Validate", "description": "True = the worker pipeline name is valid.\nFalse = the worker pipeline name is invalid. Raise an exception.", "type": "IfCondition", "dependsOn": [ { "activity": "Validate Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@bool(activity('Validate Pipeline').output.PipelineExists)", "type": "Expression" }, "ifFalseActivities": [ { "name": "Throw Exception - Invalid Infant", "description": "Throw an exception with details about the invalid worker pipeline name.", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "Throw Exception", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "Message": { "value": "@concat('Worker pipeline [',variables('WorkerPipelineName'),'] is not valid in target Data Factory [',variables('WorkerDataFactoryName'),']')", "type": "Expression" } } } }, { "name": "Update Execution With Invalid Worker", "description": "Update the current execution table with an informed status for the worker pipeline that couldn't be executed.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "CallingActivity": { "value": "InvalidPipelineName", "type": "String" }, "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Log Validate Function Activity Failure", "description": "Handle true failures from calling out to the Azure Function and update the current execution table accordingly so a restart can occur.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Validate Pipeline", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" }, "CallingActivity": { "value": "ValidatePipeline", "type": "String" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Log Pipeline Validating", "description": "Sets the current pipeline with a status of validating within the current execution database table.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineValidating]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ], "parameters": { "executionId": { "type": "string" }, "stageId": { "type": "int" }, "pipelineId": { "type": "int" } }, "variables": { "WorkerPipelineState": { "type": "Boolean" }, "WorkerAppId": { "type": "String" }, "WorkerAppSecret": { "type": "String" }, "WorkerRunId": { "type": "String" }, "WorkerTenantId": { "type": "String" }, "WorkerSubscriptionId": { "type": "String" }, "WorkerPipelineName": { "type": "String" }, "WorkerDataFactoryName": { "type": "String" }, "WorkerResourceGroup": { "type": "String" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "procfwk", "Infant" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/FrameworkFunctions')]", "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", "[concat(variables('factoryId'), '/pipelines/Throw Exception')]" ] }, { "name": "[concat(parameters('factoryName'), '/Check For Running Pipeline')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "For a given pipeline and optional batch name establish if a pipeline run is already in progress. Throw an exception if it it.", "activities": [ { "name": "Get Subscription", "description": "Use the Azure Management API to return the current subscription.", "type": "WebActivity", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "url": "https://management.azure.com/subscriptions?api-version=2020-01-01", "method": "GET", "headers": {}, "authentication": { "type": "MSI", "resource": "https://management.core.windows.net/" } } }, { "name": "Get Pipeline Runs", "description": "Use the Azure Management API to return a list of pipeline runs within the given time window.", "type": "WebActivity", "dependsOn": [ { "activity": "Get Query Run Days Value", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Check for Valid Pipeline Name", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "url": { "value": "https://management.azure.com/subscriptions/@{variables('SubscriptionId')}/resourceGroups/@{activity('Get Resource Group').output.firstRow.PropertyValue}/providers/Microsoft.DataFactory/factories/@{pipeline().DataFactory}/queryPipelineRuns?api-version=2018-06-01", "type": "Expression" }, "method": "POST", "headers": {}, "body": { "value": "{\n \"lastUpdatedAfter\": \"@{adddays(utcnow(),int(activity('Get Query Run Days Value').output.firstRow.PropertyValue))}\",\n \"lastUpdatedBefore\": \"@{utcnow()}\",\n \"filters\": [\n {\n \"operand\": \"PipelineName\",\n \"operator\": \"Equals\",\n \"values\": [\n \"@{pipeline().parameters.PipelineName}\"\n ]\n }\n ]\n}", "type": "Expression" }, "authentication": { "type": "MSI", "resource": "https://management.core.windows.net/" } } }, { "name": "Set Parsed Subscription", "description": "Parse the subscription string value to return just the subscription ID.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Subscription", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "SubscriptionId", "value": { "value": "@replace(activity('Get Subscription').output.value[0].id,'/subscriptions/','')", "type": "Expression" } } }, { "name": "Filter Running Pipelines", "description": "Filter the pipeline runs results for pipelines that exclude the current triggered run and that are currently running (in progress or queued).", "type": "Filter", "dependsOn": [ { "activity": "Get Pipeline Runs", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Pipeline Runs').output.value", "type": "Expression" }, "condition": { "value": "@and(not(equals(item().runId,pipeline().parameters.ThisRunId)),or(equals(item().status,'InProgress'),equals(item().status,'Queued')))", "type": "Expression" } } }, { "name": "Get Resource Group", "description": "Using the metadata properties table return the framework factory resource group name.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "FrameworkFactoryResourceGroup" } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Get Query Run Days Value", "description": "Using the metadata properties table return the run days value to provide the API request with a date range for pipeline executions.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "PreviousPipelineRunsQueryRange" } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "If Pipeline Is Running", "description": "If the running pipeline count is greater than or equal to one.\nTrue = raise an exception.", "type": "IfCondition", "dependsOn": [ { "activity": "If Using Batch Executions", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@greaterOrEquals(int(variables('RunCount')),1)", "type": "Expression" }, "ifTrueActivities": [ { "name": "Throw Exceptiion - Pipeline Running", "description": "Using the utils pipeline raise an exception to stop the new trigger while a run is already in progress.", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "Throw Exception", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "Message": { "value": "@concat('Provided pipeline name (',pipeline().parameters.PipelineName,') still has a run in progress or queued given the query range parameters set in the properties table.')", "type": "Expression" } } } } ] } }, { "name": "Get Execution Batch Status", "description": "Using the metadata properties table return the flag to indicate if batch execution setting are enabled or disabled.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "UseExecutionBatches" } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "If Using Batch Executions", "description": "True = batch executions are enabled.\nFalse = batch execution are disabled.", "type": "IfCondition", "dependsOn": [ { "activity": "Get Execution Batch Status", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Filter Running Pipelines", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@equals(activity('Get Execution Batch Status').output.firstRow.PropertyValue,string(1))", "type": "Expression" }, "ifFalseActivities": [ { "name": "Set Run Count Without Batch", "description": "Set the pipelines running count variable to be tested later.", "type": "SetVariable", "dependsOn": [], "userProperties": [], "typeProperties": { "variableName": "RunCount", "value": { "value": "@string(activity('Filter Running Pipelines').output.FilteredItemsCount)", "type": "Expression" } } } ], "ifTrueActivities": [ { "name": "Filter for Batch Name", "description": "Further filter the return pipeline runs for any running pipelines with the same batch name value.", "type": "Filter", "dependsOn": [], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Filter Running Pipelines').output.value", "type": "Expression" }, "condition": { "value": "@equals(item().parameters.BatchName,pipeline().parameters.BatchName)", "type": "Expression" } } }, { "name": "Set Run Count for Batch", "description": "Set the resulting pipeline running count variable to be tested later.", "type": "SetVariable", "dependsOn": [ { "activity": "Filter for Batch Name", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "RunCount", "value": { "value": "@string(activity('Filter for Batch Name').output.FilteredItemsCount)", "type": "Expression" } } } ] } }, { "name": "Check for Valid Pipeline Name", "description": "Use the Azure Management API to return and establish if the framework pipeline exists in the target Data Factory, including being deployed.", "type": "WebActivity", "dependsOn": [ { "activity": "Set Parsed Subscription", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Resource Group", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "url": { "value": "https://management.azure.com/subscriptions/@{variables('SubscriptionId')}/resourceGroups/@{activity('Get Resource Group').output.firstRow.PropertyValue}/providers/Microsoft.DataFactory/factories/@{pipeline().DataFactory}/pipelines/@{pipeline().parameters.PipelineName}?api-version=2018-06-01", "type": "Expression" }, "method": "GET", "headers": {}, "authentication": { "type": "MSI", "resource": "https://management.core.windows.net/" } } } ], "parameters": { "BatchName": { "type": "string", "defaultValue": "NotUsed" }, "PipelineName": { "type": "string" }, "ThisRunId": { "type": "string" } }, "variables": { "SubscriptionId": { "type": "String" }, "RunCount": { "type": "String" } }, "folder": { "name": "_ProcFwk/_ProcFwkUtils" }, "annotations": [ "procfwk", "Utils" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/pipelines/Throw Exception')]" ] }, { "name": "[concat(parameters('factoryName'), '/Email Sender')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Provide a simple abstract over the send email function with request body item exposed as pipeline parameters.", "activities": [ { "name": "Send Email", "description": "Use an Azure Function to perform an SMTP client email send operation.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "SendEmail", "method": "POST", "headers": {}, "body": { "value": "{\n\"emailRecipients\": \"@{pipeline().parameters.Recipients}\",\n\"emailCcRecipients\": \"@{pipeline().parameters.CcRecipients}\",\n\"emailBccRecipients\": \"@{pipeline().parameters.BccRecipients}\",\n\"emailSubject\": \"@{pipeline().parameters.Subject}\",\n\"emailBody\": \"@{pipeline().parameters.Body}\",\n\"emailImportance\": \"@{pipeline().parameters.Importance}\"\n}", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } } ], "parameters": { "Recipients": { "type": "string" }, "CcRecipients": { "type": "string" }, "BccRecipients": { "type": "string" }, "Subject": { "type": "string" }, "Body": { "type": "string" }, "Importance": { "type": "string" } }, "folder": { "name": "_ProcFwk/_ProcFwkUtils" }, "annotations": [ "procfwk", "Utils" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/FrameworkFunctions')]" ] }, { "name": "[concat(parameters('factoryName'), '/Intentional Error')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait1", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } }, { "name": "Raise Errors or Not", "description": "Framework development worker simulator.", "type": "IfCondition", "dependsOn": [ { "activity": "Wait1", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@equals(pipeline().parameters.RaiseErrors,'true')", "type": "Expression" }, "ifTrueActivities": [ { "name": "Call Fail Procedure", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[dbo].[FailProcedure]", "storedProcedureParameters": { "RaiseError": { "value": { "value": "@pipeline().parameters.RaiseErrors", "type": "Expression" }, "type": "String" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } } ], "parameters": { "RaiseErrors": { "type": "string", "defaultValue": "false" }, "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/Throw Exception')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Provide a simple way of throwing an exception within Data Factory using TSQL error handling.", "activities": [ { "name": "Raise Error", "description": "Using a SQL database to raise an error/exception but wrapped up as a data factory pipeline. Error message information exposed as a pipeline parameter.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderQuery": { "value": "RAISERROR('@{pipeline().parameters.Message}',16,1);", "type": "Expression" }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } } ], "parameters": { "Message": { "type": "string" } }, "folder": { "name": "_ProcFwk/_ProcFwkUtils" }, "annotations": [ "procfwk", "Utils" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]" ] }, { "name": "[concat(parameters('factoryName'), '/Wait 1')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait1", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 10')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait10", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 2')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait2", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 3')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait3", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 4')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait4", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 5')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait5", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 6')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait6", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 7')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait7", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 8')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait8", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 9')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the ADF.procfwk has something to call during development.", "activities": [ { "name": "Wait9", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 15 } }, "folder": { "name": "_Workers" }, "annotations": [ "Worker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/GetSetMetadata')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "description": "Single generic dataset used to get and set all database metadata.", "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" }, "folder": { "name": "_ProcFwk" }, "annotations": [ "ADF.procfwk" ], "type": "AzureSqlTable", "schema": [], "typeProperties": {} }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/FrameworkFunctions')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Interact with the Azure Functions App used as middle ware when making requests to Worker pipelines. Authentication done at the Function App level.", "annotations": [ "ADF.procfwk" ], "type": "AzureFunction", "typeProperties": { "functionAppUrl": "[parameters('FrameworkFunctions_properties_typeProperties_functionAppUrl')]", "functionKey": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "FrameworkFunctionsKey" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/Keys')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection to Key Vault for all other ADF linked service credentials required to run the processing framework.", "annotations": [ "ADF.procfwk" ], "type": "AzureKeyVault", "typeProperties": { "baseUrl": "[parameters('Keys_properties_typeProperties_baseUrl')]" } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/SupportDatabase')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection between ADF and processing framework metadata SQLDB.", "annotations": [ "ADF.procfwk" ], "type": "AzureSqlDatabase", "typeProperties": { "connectionString": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "[parameters('SupportDatabase_properties_typeProperties_connectionString_secretName')]" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/FunctionalTestingTrigger')]", "type": "Microsoft.DataFactory/factories/triggers", "apiVersion": "2018-06-01", "properties": { "description": "Used for functional testing of the framework in a dedicated environment.", "annotations": [ "ADF.procfwk" ], "runtimeState": "Stopped", "pipelines": [ { "pipelineReference": { "referenceName": "01-Grandparent", "type": "PipelineReference" }, "parameters": {} } ], "type": "ScheduleTrigger", "typeProperties": { "recurrence": { "frequency": "Hour", "interval": 2, "startTime": "2020-04-06T15:00:00Z", "timeZone": "UTC" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/pipelines/01-Grandparent')]" ] } ] } ================================================ FILE: ARM Templates/Data Factory/v2.0 Export.json ================================================ { "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { "factoryName": { "type": "string", "metadata": "Data Factory name", "defaultValue": "" }, "FrameworkFunctions_properties_typeProperties_functionAppUrl": { "type": "string", "defaultValue": "" }, "Keys_properties_typeProperties_baseUrl": { "type": "string", "defaultValue": "" }, "SupportDatabase_properties_typeProperties_connectionString_secretName": { "type": "string", "defaultValue": "" } }, "variables": { "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" }, "resources": [ { "name": "[concat(parameters('factoryName'), '/01-Grandparent')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "procfwk grandparent pipeline used optionally to bootstrap any wider processes in your Data Factory that then calls the processing framework.", "activities": [ { "name": "procfwk", "description": "Call procfwk", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "02-Parent", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "BatchName": { "value": "@pipeline().parameters.BatchName", "type": "Expression" } } } } ], "parameters": { "BatchName": { "type": "string", "defaultValue": "NotUsed" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "procfwk", "Grandparent" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/pipelines/02-Parent')]" ] }, { "name": "[concat(parameters('factoryName'), '/02-Parent')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "ADF.procfwk parent pipeline used to bootstrap the orchestration framework in perform the first level ForEach calls in sequence for the metadata stages.", "activities": [ { "name": "Get Stages", "description": "Returns a distinct list of execution stages within the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Set Execution Id", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetStages]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@variables('ExecutionId')", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Stages", "description": "Top level ForEach to sequentially call all processing stages within the framework metadata. Items for iteration passed from the Get Stages lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Stages", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Stages').output.value", "type": "Expression" }, "isSequential": true, "activities": [ { "name": "Stage Executor", "description": "Call to the framework generic child pipeline for a given execution stage.", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Log Stage Preparing", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "03-Child", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "StageId": { "value": "@item().StageId", "type": "Expression" }, "ExecutionId": { "value": "@variables('ExecutionId')", "type": "Expression" } } } }, { "name": "Log Stage Preparing", "description": "Update the current execution table flagging all pipelines within the stage as preparing.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Check and Update Blockers", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogStagePreparing]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@variables('ExecutionId')", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check and Update Blockers", "description": "Used to double check and stop the next execution stage if failures and blockers have be incurred. This also depends on the failure handling property value which defines the stored procedure behaviour.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[CheckForBlockedPipelines]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@variables('ExecutionId')", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Execution Wrapper", "description": "Wrapper to reset and restart processing or create a completely new execution instance of the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Clean Up Previous Run", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[ExecutionWrapper]", "storedProcedureParameters": { "CallingOrchestratorName": { "type": "String", "value": { "value": "@pipeline().DataFactory", "type": "Expression" } }, "BatchName": { "type": "String", "value": { "value": "@pipeline().parameters.BatchName", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Check Outcome and Update Logs", "description": "After a successful execution run the current execution metadata is moved to the long term logging table by this stored procedure call. Otherwise an error will be raised.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Stages", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[UpdateExecutionLog]", "storedProcedureParameters": { "PerformErrorCheck": { "value": { "value": "@bool(1)", "type": "Expression" }, "type": "Boolean" }, "ExecutionId": { "value": { "value": "@variables('ExecutionId')", "type": "Expression" }, "type": "Guid" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check Previous Execution", "description": "Query the current execution table for worker pipelines that require a clean up from the previous execution run.", "type": "Lookup", "dependsOn": [ { "activity": "Execute Precursor", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[CheckPreviousExeuction]", "storedProcedureParameters": { "BatchName": { "type": "String", "value": { "value": "@pipeline().parameters.BatchName", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Clean Up Previous Run", "description": "Handle Worker pipelines that are reported as Running when the parent pipeline is called again. Get what the actual status of those pipelines is.", "type": "ForEach", "dependsOn": [ { "activity": "Check Previous Execution", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Check Metadata Integrity", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Check Previous Execution').output.value", "type": "Expression" }, "isSequential": false, "batchCount": 50, "activities": [ { "name": "Get SPN Details", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": true, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetWorkerAuthDetails]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@item().LocalExecutionId", "type": "Expression" } }, "PipelineId": { "type": "Int32", "value": { "value": "@item().PipelineId", "type": "Expression" } }, "StageId": { "type": "Int32", "value": { "value": "@item().StageId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Pipeline Checking", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineChecking]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Pipeline Status", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Get SPN Details", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Log Pipeline Checking", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "CheckPipelineStatus", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',activity('Get SPN Details').output.firstRow.TenantId,'\",\n \"applicationId\": \"',activity('Get SPN Details').output.firstRow.AppId,'\",\n \"authenticationKey\": \"',activity('Get SPN Details').output.firstRow.AppSecret,'\",\n \"subscriptionId\": \"',activity('Get SPN Details').output.firstRow.SubscriptionId,'\",\n \"resourceGroupName\": \"',item().ResourceGroupName,'\",\n \"orchestratorName\": \"',item().OrchestratorName,'\",\n \"orchestratorType\": \"',item().OrchestratorType,'\",\n \"pipelineName\": \"',item().PipelineName,'\",\n \"runId\": \"',item().PipelineRunId,'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Set Pipeline Status", "description": "Update the metadata depending on the actual pipeline outcome. Using the status as the case.", "type": "Switch", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@activity('Get Pipeline Status').output.ActualStatus", "type": "Expression" }, "cases": [ { "value": "Failed", "activities": [ { "name": "Pipeline Status Failed", "description": "Updates the current execution table with a pipeline status of failed if the function outcome is failed. Also blocks pipelines in the downstream execution stage.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": null, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Succeeded", "activities": [ { "name": "Pipeline Status Succeeded", "description": "Updates the current execution table with a pipeline status of success if the function outcome is succeeded.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Queued", "activities": [ { "name": "Pipeline Status Queued - Running", "description": "Updates the current execution table with a pipeline status of running if the function outcome is queued.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "InProgress", "activities": [ { "name": "Pipeline Status InProgress - Running", "description": "Updates the current execution table with a pipeline status of running if the function outcome is in progress.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Cancelled", "activities": [ { "name": "Pipeline Status Cancelled", "description": "Updates the current execution table with a pipeline status of cancelled if the function outcome is cancelled.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineCancelled]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" }, "CleanUpRun": { "value": { "value": "@bool(1)", "type": "Expression" }, "type": "Boolean" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } ], "defaultActivities": [ { "name": "Pipeline Status Unknown", "description": "Updates the current execution table with a pipeline status of unknown if the function returns an unexpected outcome.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineUnknown]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" }, "CleanUpRun": { "value": { "value": "@bool(1)", "type": "Expression" }, "type": "Boolean" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Set Last Check DateTime", "description": "Update the current execution table with a date time from when the function last checked the pipeline status.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineLastStatusCheck]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Execute Precursor", "description": "Uses the database property value ExecutionPrecursorProc to run any custom logic against the metadata database before the execution run starts.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Is Parent Already Running", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[ExecutePrecursorProcedure]" }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Set Execution Id", "description": "Set the local execution Id to a pipeline variable for each in several downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Execution Wrapper", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "ExecutionId", "value": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" } } }, { "name": "Is Parent Already Running", "description": "Establish before anything else if the parent pipeline is already running. Batch execution aware.", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "Check For Running Pipeline", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "BatchName": { "value": "@pipeline().parameters.BatchName", "type": "Expression" }, "PipelineName": { "value": "@pipeline().Pipeline", "type": "Expression" }, "ThisRunId": { "value": "@pipeline().RunId", "type": "Expression" } } } }, { "name": "Check Metadata Integrity", "description": "Performs a series of checks on all metadata held in the framework SQLDB. This is intended to raise errors before an execution run even starts.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Precursor", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[CheckMetadataIntegrity]", "storedProcedureParameters": { "BatchName": { "value": { "value": "@pipeline().parameters.BatchName", "type": "Expression" }, "type": "String" }, "DebugMode": { "value": { "value": "@bool(0)", "type": "Expression" }, "type": "Boolean" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ], "parameters": { "BatchName": { "type": "string", "defaultValue": "NotUsed" } }, "variables": { "ExecutionId": { "type": "String" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "procfwk", "Parent" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", "[concat(variables('factoryId'), '/pipelines/Check For Running Pipeline')]", "[concat(variables('factoryId'), '/pipelines/03-Child')]", "[concat(variables('factoryId'), '/linkedServices/FrameworkFunctions')]" ] }, { "name": "[concat(parameters('factoryName'), '/03-Child')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "procfwk child pipeline used to execute Worker pipelines within a given execution stage. This pipeline will be called once for each stage, then execute all Workers in parallel.", "activities": [ { "name": "Get Pipelines", "description": "Returns all pipelines from the metadata to be executed within a given processing stage.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelinesInStage]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" } }, "StageId": { "type": "Int32", "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Pipelines", "description": "Second level ForEach to run in parallel all pipelines within the stage. Items for iteration passed from the Get Pipelines lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Pipelines", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Pipelines').output.value", "type": "Expression" }, "isSequential": false, "batchCount": 50, "activities": [ { "name": "Worker Pipeline Executor", "description": "Run the required worker pipeline and wait for its completion. Update metadata once done.", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "04-Infant", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "executionId": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "stageId": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "pipelineId": { "value": "@item().PipelineId", "type": "Expression" } } } } ] } } ], "parameters": { "StageId": { "type": "int" }, "ExecutionId": { "type": "string" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "procfwk", "Child" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/pipelines/04-Infant')]" ] }, { "name": "[concat(parameters('factoryName'), '/04-Infant')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "procfwk infant pipeline used to check when the processing pipeline called by the Child completes and passes the resulting status back to the metadata database.", "activities": [ { "name": "Execute Worker Pipeline", "description": "The lowest level executor with the metadata framework to call existing processing pipelines within Data Factory. The function called will block processing and wait for an outcome.", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Log Pipeline Running", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Pipeline Params", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "ExecutePipeline", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',variables('WorkerCoreDetails')[0].tenantId,'\",\n \"applicationId\": \"',variables('WorkerCoreDetails')[0].applicationId,'\",\n \"authenticationKey\": \"',variables('WorkerCoreDetails')[0].authenticationKey,'\",\n \"subscriptionId\": \"',variables('WorkerCoreDetails')[0].subscriptionId,'\",\n \"resourceGroupName\": \"',variables('WorkerCoreDetails')[0].resourceGroupName,'\",\n\t\"orchestratorName\": \"',variables('WorkerCoreDetails')[0].orchestratorName,'\",\n \"orchestratorType\": \"',variables('WorkerCoreDetails')[0].orchestratorType,'\",\n \"pipelineName\": \"',variables('WorkerCoreDetails')[0].pipelineName,'\"',activity('Get Pipeline Params').output.firstRow.Params,'\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Get Pipeline Params", "description": "Returns any parameters from metadata required for the processing pipeline being called. The output can be an empty string if no parameters are required.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelineParameters]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Pipeline Running", "description": "Sets the current pipeline with a status of running within the current execution database table.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Is Target Worker Validate", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Log Execute Function Activity Failure", "description": "Handle true failures from calling out to the Azure Function and update the current execution table accordingly so a restart can occur.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Worker Pipeline", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" }, "CallingActivity": { "value": "ExecuteWorkerPipeline", "type": "String" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Update Run Id", "description": "Provide the actual ADF run ID back to the current execution table for long term logging and alignment between the metadata other Azure monitoring tools.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Set Run Id", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunId]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@variables('WorkerRunId')", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check For Alerts", "description": "Checks the properties tables and if any recipients in the database require alerts sending for the current pipeline ID.", "type": "Lookup", "dependsOn": [ { "activity": "Update Run Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Pipeline Result", "dependencyConditions": [ "Completed" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[CheckForEmailAlerts]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Send Alerts", "description": "True = alerts need sending.\nFalse = do nothing.", "type": "IfCondition", "dependsOn": [ { "activity": "Check For Alerts", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@activity('Check For Alerts').output.firstRow.SendAlerts", "type": "Expression" }, "ifTrueActivities": [ { "name": "Get Email Parts", "description": "Return all required content from the metadata database to send an email alerting using the procfwk. The lookup returns the exact content for the function body request.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": true, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetEmailAlertParts]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Call Email Sender", "description": "Pass off email request to Utils Send Email pipeline.", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Get Email Parts", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "Email Sender", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "Recipients": { "value": "@activity('Get Email Parts').output.firstRow.emailRecipients", "type": "Expression" }, "CcRecipients": { "value": "@activity('Get Email Parts').output.firstRow.emailCcRecipients", "type": "Expression" }, "BccRecipients": { "value": "@activity('Get Email Parts').output.firstRow.emailBccRecipients", "type": "Expression" }, "Subject": { "value": "@activity('Get Email Parts').output.firstRow.emailSubject", "type": "Expression" }, "Body": { "value": "@activity('Get Email Parts').output.firstRow.emailBody", "type": "Expression" }, "Importance": { "value": "@activity('Get Email Parts').output.firstRow.emailImportance", "type": "Expression" } } } } ] } }, { "name": "Wait Until Pipeline Completes", "description": "Loops until the Worker pipeline called completes.\n\nSimple status:\n- Running = new iteration.\n- Done = break.", "type": "Until", "dependsOn": [ { "activity": "Get Wait Duration", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Execute Worker Pipeline", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Run Id", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@variables('WorkerPipelineState')", "type": "Expression" }, "activities": [ { "name": "Get Worker Pipeline Status", "description": "Checks the status of a given processing pipeline and provides the value for the downstream framework activities to act upon.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "CheckPipelineStatus", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',variables('WorkerCoreDetails')[0].tenantId,'\",\n \"applicationId\": \"',variables('WorkerCoreDetails')[0].applicationId,'\",\n \"authenticationKey\": \"',variables('WorkerCoreDetails')[0].authenticationKey,'\",\n \"subscriptionId\": \"',variables('WorkerCoreDetails')[0].subscriptionId,'\",\n \"resourceGroupName\": \"',variables('WorkerCoreDetails')[0].resourceGroupName,'\",\n\t\"orchestratorName\": \"',variables('WorkerCoreDetails')[0].orchestratorName,'\",\n \"orchestratorType\": \"',variables('WorkerCoreDetails')[0].orchestratorType,'\",\n \"pipelineName\": \"',variables('WorkerCoreDetails')[0].pipelineName,'\",\n \"runId\": \"',variables('WorkerRunId'),'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Wait If Running", "description": "True = Do nothing.\nFalse = Wait, before the next iteration.", "type": "IfCondition", "dependsOn": [ { "activity": "Set Worker State", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@variables('WorkerPipelineState')", "type": "Expression" }, "ifFalseActivities": [ { "name": "Wait for Pipeline", "description": "The processing pipeline is still running so Wait before checking its status again.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@activity('Get Wait Duration').output.firstRow.PropertyValue", "type": "Expression" } } } ] } }, { "name": "Set Last Check DateTime", "description": "Update the current execution table with a date time from when the Worker pipeline status was last checked as part of the Until iterations.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Worker Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineLastStatusCheck]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Log Check Function Activity Failure", "description": "Report to the current execution table that the framework pipeline activity has failed. This failure is outside of the scope of the framework and is probably related to a wider platform problem.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Worker Pipeline Status", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "CallingActivity": { "value": "GetWorkerPipelineStatus", "type": "String" }, "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Set Worker State", "description": "Set the bool state of the Worker pipeline to be used by the Until and If expressions. True = Complete, False = Running.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerPipelineState", "value": { "value": "@equals('Complete',activity('Get Worker Pipeline Status').output.SimpleStatus)", "type": "Expression" } } } ], "timeout": "0.00:10:00" } }, { "name": "Set Pipeline Result", "description": "Receives the outcome from the function execution for a given processing pipeline and updates the current execution table with different pipelines status values depending on the result (case).", "type": "Switch", "dependsOn": [ { "activity": "Wait Until Pipeline Completes", "dependencyConditions": [ "Completed" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@activity('Get Worker Pipeline Status').output.ActualStatus", "type": "Expression" }, "cases": [ { "value": "Succeeded", "activities": [ { "name": "Pipeline Status Succeeded", "description": "Updates the current execution table with a pipeline status of success if the function outcome is succeeded.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Failed", "activities": [ { "name": "Pipeline Status Failed", "description": "Updates the current execution table with a pipeline status of failed if the function outcome is failed. Also blocks pipelines in the downstream execution stage.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@variables('WorkerRunId')", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Worker Pipeline Error Details", "description": "Get the activity error details for the run ID of the worker pipeline called. Returns an array of all errors.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "GetActivityErrors", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',variables('WorkerCoreDetails')[0].tenantId,'\",\n \"applicationId\": \"',variables('WorkerCoreDetails')[0].applicationId,'\",\n \"authenticationKey\": \"',variables('WorkerCoreDetails')[0].authenticationKey,'\",\n \"subscriptionId\": \"',variables('WorkerCoreDetails')[0].subscriptionId,'\",\n \"resourceGroupName\": \"',variables('WorkerCoreDetails')[0].resourceGroupName,'\",\n\t\"orchestratorName\": \"',variables('WorkerCoreDetails')[0].orchestratorName,'\",\n \"orchestratorType\": \"',variables('WorkerCoreDetails')[0].orchestratorType,'\",\n \"pipelineName\": \"',variables('WorkerCoreDetails')[0].pipelineName,'\",\n \"runId\": \"',variables('WorkerRunId'),'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Log Error Details", "description": "Parses pipeline error details and persists them to the metadata database error log table.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Worker Pipeline Error Details", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetErrorLogDetails]", "storedProcedureParameters": { "JsonErrorDetails": { "value": { "value": "@string(activity('Get Worker Pipeline Error Details').output)", "type": "Expression" }, "type": "String" }, "LocalExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Cancelled", "activities": [ { "name": "Pipeline Status Cancelled", "description": "Updates the current execution table with a pipeline status of cancelled if the function outcome is cancelled.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineCancelled]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } ], "defaultActivities": [ { "name": "Pipeline Status Unknown", "description": "Updates the current execution table with a pipeline status of unknown if the function returns an unexpected outcome.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineUnknown]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Get Wait Duration", "description": "Return wait duration in seconds from database properties table to be used during each Until iteration when the Worker pipeline is still running.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "PipelineStatusCheckDuration" } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Set Run Id", "description": "Set local variable from activity output once for value reuse in downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Execute Worker Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerRunId", "value": { "value": "@activity('Execute Worker Pipeline').output.RunId", "type": "Expression" } } }, { "name": "Validate Pipeline", "description": "Query the target data factory and establish if the provided worker pipeline name is valid.", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Log Pipeline Validating", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Capture Worker Core Details as an Array", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "ValidatePipeline", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',variables('WorkerCoreDetails')[0].tenantId,'\",\n \"applicationId\": \"',variables('WorkerCoreDetails')[0].applicationId,'\",\n \"authenticationKey\": \"',variables('WorkerCoreDetails')[0].authenticationKey,'\",\n \"subscriptionId\": \"',variables('WorkerCoreDetails')[0].subscriptionId,'\",\n \"resourceGroupName\": \"',variables('WorkerCoreDetails')[0].resourceGroupName,'\",\n\t\"orchestratorName\": \"',variables('WorkerCoreDetails')[0].orchestratorName,'\",\n \"orchestratorType\": \"',variables('WorkerCoreDetails')[0].orchestratorType,'\",\n \"pipelineName\": \"',variables('WorkerCoreDetails')[0].pipelineName,'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Is Target Worker Validate", "description": "True = the worker pipeline name is valid.\nFalse = the worker pipeline name is invalid. Raise an exception.", "type": "IfCondition", "dependsOn": [ { "activity": "Validate Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@bool(activity('Validate Pipeline').output.PipelineExists)", "type": "Expression" }, "ifFalseActivities": [ { "name": "Throw Exception - Invalid Infant", "description": "Throw an exception with details about the invalid worker pipeline name.", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "Throw Exception", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "Message": { "value": "@concat('Worker pipeline [',variables('WorkerCoreDetails')[0].pipelineName,'] is not valid in target Orchestrator [',variables('WorkerCoreDetails')[0].orchestratorName,']')", "type": "Expression" } } } }, { "name": "Update Execution With Invalid Worker", "description": "Update the current execution table with an informed status for the worker pipeline that couldn't be executed.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "CallingActivity": { "value": "InvalidPipelineName", "type": "String" }, "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Log Validate Function Activity Failure", "description": "Handle true failures from calling out to the Azure Function and update the current execution table accordingly so a restart can occur.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Validate Pipeline", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" }, "CallingActivity": { "value": "ValidatePipeline", "type": "String" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Log Pipeline Validating", "description": "Sets the current pipeline with a status of validating within the current execution database table.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineValidating]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Worker Core Details", "description": "Return worker pipeline information for metadata database. Including target data factory, pipeline name and resource group. Return the SPN ID and Secret for the worker pipeline being executed. Called at this level as each pipeline can have a different SPN.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": true, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetWorkerDetailsWrapper]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" } }, "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } }, "StageId": { "type": "Int32", "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Capture Worker Core Details as an Array", "description": "Add all worker pipeline details to a local variable array that can be accessed by each function call requiring the values.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Core Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerCoreDetails", "value": { "value": "@array(activity('Get Worker Core Details').output.firstRow)", "type": "Expression" } } } ], "parameters": { "executionId": { "type": "string" }, "stageId": { "type": "int" }, "pipelineId": { "type": "int" } }, "variables": { "WorkerPipelineState": { "type": "Boolean" }, "WorkerRunId": { "type": "String" }, "WorkerCoreDetails": { "type": "Array" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "procfwk", "Infant" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/FrameworkFunctions')]", "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", "[concat(variables('factoryId'), '/pipelines/Email Sender')]", "[concat(variables('factoryId'), '/pipelines/Throw Exception')]" ] }, { "name": "[concat(parameters('factoryName'), '/Check For Running Pipeline')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "For a given pipeline and optional batch name establish if a pipeline run is already in progress. Throw an exception if it it.", "activities": [ { "name": "Filter Running Pipelines", "description": "Filter the pipeline runs results for pipelines that exclude the current triggered run and that are currently running (in progress or queued).", "type": "Filter", "dependsOn": [ { "activity": "Switch For Orchestrator Type", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@variables('PipelineRuns')", "type": "Expression" }, "condition": { "value": "@and(not(equals(item().runId,pipeline().parameters.ThisRunId)),or(equals(item().status,'InProgress'),equals(item().status,'Queued')))", "type": "Expression" } } }, { "name": "Get Framework Orchestrator Details", "description": "Using the metadata orchestrators return details about the resource running the framework pipelines.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetFrameworkOrchestratorDetails]", "storedProcedureParameters": { "CallingOrchestratorName": { "type": "String", "value": { "value": "@pipeline().DataFactory", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Get Query Run Days Value", "description": "Using the metadata properties table return the run days value to provide the API request with a date range for pipeline executions.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "PreviousPipelineRunsQueryRange" } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "If Pipeline Is Running", "description": "If the running pipeline count is greater than or equal to one.\nTrue = raise an exception.", "type": "IfCondition", "dependsOn": [ { "activity": "If Using Batch Executions", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@greaterOrEquals(int(variables('RunCount')),1)", "type": "Expression" }, "ifTrueActivities": [ { "name": "Throw Exception - Pipeline Running", "description": "Using the utils pipeline raise an exception to stop the new trigger while a run is already in progress.", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "Throw Exception", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "Message": { "value": "@concat('Provided pipeline name (',pipeline().parameters.PipelineName,') still has a run in progress or queued given the query range parameters set in the properties table.')", "type": "Expression" } } } } ] } }, { "name": "Get Execution Batch Status", "description": "Using the metadata properties table return the flag to indicate if batch execution setting are enabled or disabled.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "UseExecutionBatches" } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "If Using Batch Executions", "description": "True = batch executions are enabled.\nFalse = batch execution are disabled.", "type": "IfCondition", "dependsOn": [ { "activity": "Get Execution Batch Status", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Filter Running Pipelines", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@equals(activity('Get Execution Batch Status').output.firstRow.PropertyValue,string(1))", "type": "Expression" }, "ifFalseActivities": [ { "name": "Set Run Count Without Batch", "description": "Set the pipelines running count variable to be tested later.", "type": "SetVariable", "dependsOn": [], "userProperties": [], "typeProperties": { "variableName": "RunCount", "value": { "value": "@string(activity('Filter Running Pipelines').output.FilteredItemsCount)", "type": "Expression" } } } ], "ifTrueActivities": [ { "name": "Filter for Batch Name", "description": "Further filter the return pipeline runs for any running pipelines with the same batch name value.", "type": "Filter", "dependsOn": [], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Filter Running Pipelines').output.value", "type": "Expression" }, "condition": { "value": "@equals(item().parameters.BatchName,pipeline().parameters.BatchName)", "type": "Expression" } } }, { "name": "Set Run Count for Batch", "description": "Set the resulting pipeline running count variable to be tested later.", "type": "SetVariable", "dependsOn": [ { "activity": "Filter for Batch Name", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "RunCount", "value": { "value": "@string(activity('Filter for Batch Name').output.FilteredItemsCount)", "type": "Expression" } } } ] } }, { "name": "Set Subscription Id", "description": "Set the subscription Id value to a local variable for use in various downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Framework Orchestrator Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "SubscriptionId", "value": { "value": "@activity('Get Framework Orchestrator Details').output.firstRow.SubscriptionId", "type": "Expression" } } }, { "name": "Set Resource Group Name", "description": "Set the resource group name value to a local variable for use in various downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Framework Orchestrator Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "ResourceGroupName", "value": { "value": "@activity('Get Framework Orchestrator Details').output.firstRow.ResourceGroupName", "type": "Expression" } } }, { "name": "Set Orchestrator Type", "description": "Set the orchestrator type value to a local variable for use in various downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Framework Orchestrator Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "OrchestratorType", "value": { "value": "@toUpper(activity('Get Framework Orchestrator Details').output.firstRow.OrchestratorType)", "type": "Expression" } } }, { "name": "Switch For Orchestrator Type", "description": "Switch and handle requests for both Azure Data Factory (ADF) and Azure Synapse Analytics (SYN).", "type": "Switch", "dependsOn": [ { "activity": "Set Orchestrator Type", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Query Run Days", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Resource Group Name", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Subscription Id", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@variables('OrchestratorType')", "type": "Expression" }, "cases": [ { "value": "ADF", "activities": [ { "name": "Check for Valid ADF Pipeline Name", "description": "Use the Azure Management API to return and establish if the framework pipeline exists in the target Data Factory instance, including being deployed.", "type": "WebActivity", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "url": { "value": "https://management.azure.com/subscriptions/@{variables('SubscriptionId')}/resourceGroups/@{variables('ResourceGroupName')}/providers/Microsoft.DataFactory/factories/@{pipeline().DataFactory}/pipelines/@{pipeline().parameters.PipelineName}?api-version=2018-06-01", "type": "Expression" }, "method": "GET", "headers": {}, "authentication": { "type": "MSI", "resource": "https://management.core.windows.net/" } } }, { "name": "Get ADF Pipeline Runs", "description": "Use the Azure Management API to return a list of data factory pipeline runs within the given time window.", "type": "WebActivity", "dependsOn": [ { "activity": "Check for Valid ADF Pipeline Name", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "url": { "value": "https://management.azure.com/subscriptions/@{variables('SubscriptionId')}/resourceGroups/@{variables('ResourceGroupName')}/providers/Microsoft.DataFactory/factories/@{pipeline().DataFactory}/queryPipelineRuns?api-version=2018-06-01", "type": "Expression" }, "method": "POST", "headers": {}, "body": { "value": "{\n \"lastUpdatedAfter\": \"@{adddays(utcnow(),int(variables('QueryRunDays')))}\",\n \"lastUpdatedBefore\": \"@{utcnow()}\",\n \"filters\": [\n {\n \"operand\": \"PipelineName\",\n \"operator\": \"Equals\",\n \"values\": [\n \"@{pipeline().parameters.PipelineName}\"\n ]\n }\n ]\n}", "type": "Expression" }, "authentication": { "type": "MSI", "resource": "https://management.core.windows.net/" } } }, { "name": "Set ADF Runs Output", "description": "Set output to local array for use in downstream filtering and pipeline checks. Use the same array output for both switch cases.", "type": "SetVariable", "dependsOn": [ { "activity": "Get ADF Pipeline Runs", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "PipelineRuns", "value": { "value": "@activity('Get ADF Pipeline Runs').output.value", "type": "Expression" } } } ] }, { "value": "SYN", "activities": [ { "name": "Check for Valid SYN Pipeline Name", "description": "Use the Azure Management API to return and establish if the framework pipeline exists in the target Synapse instance, including being deployed.\n\nSee: https://docs.microsoft.com/en-us/rest/api/synapse/data-plane/pipeline/getpipeline", "type": "WebActivity", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "url": { "value": "https://@{pipeline().DataFactory}.dev.azuresynapse.net/pipelines/@{pipeline().parameters.PipelineName}?api-version=2019-06-01-preview", "type": "Expression" }, "method": "GET", "headers": {}, "authentication": { "type": "MSI", "resource": "https://management.core.windows.net/" } } }, { "name": "Get SYN Pipeline Runs", "description": "Use the Azure Management API to return a list of synapse pipeline runs within the given time window.\n\nSee: https://docs.microsoft.com/en-us/rest/api/synapse/data-plane/pipelinerun/querypipelinerunsbyworkspace", "type": "WebActivity", "dependsOn": [ { "activity": "Check for Valid SYN Pipeline Name", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "url": { "value": "https://@{pipeline().DataFactory}.dev.azuresynapse.net/queryPipelineRuns?api-version=2019-06-01-preview", "type": "Expression" }, "method": "POST", "headers": {}, "body": { "value": "{\n \"lastUpdatedAfter\": \"@{adddays(utcnow(),int(variables('QueryRunDays')))}\",\n \"lastUpdatedBefore\": \"@{utcnow()}\",\n \"filters\": [\n {\n \"operand\": \"PipelineName\",\n \"operator\": \"Equals\",\n \"values\": [\n \"@{pipeline().parameters.PipelineName}\"\n ]\n }\n ]\n}", "type": "Expression" }, "authentication": { "type": "MSI", "resource": "https://management.core.windows.net/" } } }, { "name": "Set SYN Runs Output", "description": "Set output to local array for use in downstream filtering and pipeline checks. Use the same array output for both switch cases.", "type": "SetVariable", "dependsOn": [ { "activity": "Get SYN Pipeline Runs", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "PipelineRuns", "value": { "value": "@activity('Get SYN Pipeline Runs').output.value", "type": "Expression" } } } ] } ], "defaultActivities": [ { "name": "Throw Exception Invalid Orchestrator Type", "description": "Throw exception if switch cases are not met.", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "Throw Exception", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "Message": "Invalid orchestrator type provided. Unable to check pipeline running state." } } } ] } }, { "name": "Set Query Run Days", "description": "Set the query run days value to a local variable for use in various downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Query Run Days Value", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "QueryRunDays", "value": { "value": "@activity('Get Query Run Days Value').output.firstRow.PropertyValue", "type": "Expression" } } } ], "parameters": { "BatchName": { "type": "string", "defaultValue": "NotUsed" }, "PipelineName": { "type": "string" }, "ThisRunId": { "type": "string" } }, "variables": { "SubscriptionId": { "type": "String" }, "RunCount": { "type": "String" }, "ResourceGroupName": { "type": "String" }, "OrchestratorType": { "type": "String" }, "QueryRunDays": { "type": "String" }, "PipelineRuns": { "type": "Array" } }, "folder": { "name": "_ProcFwk/_ProcFwkUtils" }, "annotations": [ "procfwk", "Utils" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/pipelines/Throw Exception')]" ] }, { "name": "[concat(parameters('factoryName'), '/Email Sender')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Provide a simple abstract over the send email function with request body item exposed as pipeline parameters.", "activities": [ { "name": "Send Email", "description": "Use an Azure Function to perform an SMTP client email send operation.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "SendEmail", "method": "POST", "headers": {}, "body": { "value": "{\n\"emailRecipients\": \"@{pipeline().parameters.Recipients}\",\n\"emailCcRecipients\": \"@{pipeline().parameters.CcRecipients}\",\n\"emailBccRecipients\": \"@{pipeline().parameters.BccRecipients}\",\n\"emailSubject\": \"@{pipeline().parameters.Subject}\",\n\"emailBody\": \"@{pipeline().parameters.Body}\",\n\"emailImportance\": \"@{pipeline().parameters.Importance}\"\n}", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } } ], "parameters": { "Recipients": { "type": "string" }, "CcRecipients": { "type": "string" }, "BccRecipients": { "type": "string" }, "Subject": { "type": "string" }, "Body": { "type": "string" }, "Importance": { "type": "string" } }, "folder": { "name": "_ProcFwk/_ProcFwkUtils" }, "annotations": [ "procfwk", "Utils" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/FrameworkFunctions')]" ] }, { "name": "[concat(parameters('factoryName'), '/Intentional Error')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait1", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } }, { "name": "Raise Errors or Not", "description": "Framework development worker simulator.", "type": "IfCondition", "dependsOn": [ { "activity": "Wait1", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@equals(pipeline().parameters.RaiseErrors,'true')", "type": "Expression" }, "ifTrueActivities": [ { "name": "Call Fail Procedure", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[dbo].[FailProcedure]", "storedProcedureParameters": { "RaiseError": { "value": { "value": "@pipeline().parameters.RaiseErrors", "type": "Expression" }, "type": "String" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } } ], "parameters": { "RaiseErrors": { "type": "string", "defaultValue": "false" }, "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/Throw Exception')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Provide a simple way of throwing an exception within Data Factory using TSQL error handling.", "activities": [ { "name": "Raise Error", "description": "Using a SQL database to raise an error/exception but wrapped up as a data factory pipeline. Error message information exposed as a pipeline parameter.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderQuery": { "value": "RAISERROR('@{pipeline().parameters.Message}',16,1);", "type": "Expression" }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } } ], "parameters": { "Message": { "type": "string" } }, "folder": { "name": "_ProcFwk/_ProcFwkUtils" }, "annotations": [ "procfwk", "Utils" ] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]" ] }, { "name": "[concat(parameters('factoryName'), '/Wait 1')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait1", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 10')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait10", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 2')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait2", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 3')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait3", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 4')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait4", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 5')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait5", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 6')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait6", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 7')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait7", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 8')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait8", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Wait 9')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait9", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 15 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/GetSetMetadata')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "description": "Single generic dataset used to get and set all database metadata.", "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" }, "folder": { "name": "_ProcFwk" }, "annotations": [ "procfwk" ], "type": "AzureSqlTable", "schema": [], "typeProperties": {} }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('factoryName'), '/FrameworkFunctions')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Interact with the Azure Functions App used as middle ware when making requests to Worker pipelines. Authentication done at the Function App level.", "annotations": [ "procfwk" ], "type": "AzureFunction", "typeProperties": { "functionAppUrl": "[parameters('FrameworkFunctions_properties_typeProperties_functionAppUrl')]", "functionKey": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "FrameworkFunctionsKey" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/Keys')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection to Key Vault for all other ADF linked service credentials required to run the processing framework.", "annotations": [ "procfwk" ], "type": "AzureKeyVault", "typeProperties": { "baseUrl": "[parameters('Keys_properties_typeProperties_baseUrl')]" } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/SupportDatabase')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "Connection between ADF and processing framework metadata SQLDB.", "annotations": [ "procfwk" ], "type": "AzureSqlDatabase", "typeProperties": { "connectionString": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "[parameters('SupportDatabase_properties_typeProperties_connectionString_secretName')]" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('factoryName'), '/FunctionalTestingTrigger')]", "type": "Microsoft.DataFactory/factories/triggers", "apiVersion": "2018-06-01", "properties": { "description": "Used for functional testing of the framework in a dedicated environment.", "annotations": [ "procfwk" ], "runtimeState": "Stopped", "pipelines": [ { "pipelineReference": { "referenceName": "01-Grandparent", "type": "PipelineReference" }, "parameters": {} } ], "type": "ScheduleTrigger", "typeProperties": { "recurrence": { "frequency": "Hour", "interval": 2, "startTime": "2020-04-06T15:00:00Z", "timeZone": "UTC" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/pipelines/01-Grandparent')]" ] } ] } ================================================ FILE: ARM Templates/Functions App/v1.6 Export.json ================================================ { "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { "FunctionAppName": { "defaultValue": "", "type": "String" }, "SubscriptionAndRegionServerFarmLocation": { "defaultValue": "/subscriptions/{YourSubscriptionId}/resourceGroups/{YourResourceGroupName}/providers/Microsoft.Web/serverfarms/UKSouthPlan", "type": "String" } }, "variables": {}, "resources": [ { "type": "Microsoft.Web/sites", "apiVersion": "2018-11-01", "name": "[parameters('FunctionAppName')]", "location": "UK South", "kind": "functionapp", "properties": { "enabled": true, "hostNameSslStates": [ { "name": "[parameters('FunctionAppName')].azurewebsites.net", "sslState": "Disabled", "hostType": "Standard" }, { "name": "[parameters('FunctionAppName')].scm.azurewebsites.net", "sslState": "Disabled", "hostType": "Repository" } ], "serverFarmId": "[parameters('SubscriptionAndRegionServerFarmLocation')]", "reserved": false, "isXenon": false, "hyperV": false, "siteConfig": {}, "scmSiteAlsoStopped": false, "clientAffinityEnabled": false, "clientCertEnabled": false, "hostNamesDisabled": false, "containerSize": 1536, "dailyMemoryTimeQuota": 0, "httpsOnly": true, "redundancyMode": "None" } }, { "type": "Microsoft.Web/sites/config", "apiVersion": "2018-11-01", "name": "[concat(parameters('FunctionAppName'), '/web')]", "location": "UK South", "dependsOn": [ "[resourceId('Microsoft.Web/sites', parameters('FunctionAppName'))]" ], "properties": { "numberOfWorkers": -1, "defaultDocuments": [ "Default.htm", "Default.html", "Default.asp", "index.htm", "index.html", "iisstart.htm", "default.aspx", "index.php" ], "netFrameworkVersion": "v4.0", "phpVersion": "5.6", "requestTracingEnabled": false, "remoteDebuggingEnabled": false, "httpLoggingEnabled": false, "logsDirectorySizeLimit": 35, "detailedErrorLoggingEnabled": false, "publishingUsername": "$[parameters('FunctionAppName')]", "scmType": "None", "use32BitWorkerProcess": true, "webSocketsEnabled": false, "alwaysOn": false, "managedPipelineMode": "Integrated", "virtualApplications": [ { "virtualPath": "/", "physicalPath": "site\\wwwroot", "preloadEnabled": false } ], "loadBalancing": "LeastRequests", "experiments": { "rampUpRules": [] }, "autoHealEnabled": false, "cors": { "allowedOrigins": [ "https://functions.azure.com", "https://functions-staging.azure.com", "https://functions-next.azure.com" ], "supportCredentials": false }, "localMySqlEnabled": false, "ipSecurityRestrictions": [ { "ipAddress": "Any", "action": "Allow", "priority": 1, "name": "Allow all", "description": "Allow all access" } ], "scmIpSecurityRestrictions": [ { "ipAddress": "Any", "action": "Allow", "priority": 1, "name": "Allow all", "description": "Allow all access" } ], "scmIpSecurityRestrictionsUseMain": false, "http20Enabled": false, "minTlsVersion": "1.2", "ftpsState": "AllAllowed", "reservedInstanceCount": 0 } }, { "type": "Microsoft.Web/sites/deployments", "apiVersion": "2018-11-01", "name": "[concat(parameters('FunctionAppName'), '/5aee31e75b934c5f80fca0b26fe47dbb')]", "location": "UK South", "dependsOn": [ "[resourceId('Microsoft.Web/sites', parameters('FunctionAppName'))]" ], "properties": { "status": 4, "author_email": "N/A", "author": "N/A", "deployer": "Push-Deployer", "message": "Created via a push deployment", "start_time": "2020-04-30T13:29:11.8890295Z", "end_time": "2020-04-30T13:29:16.4504076Z", "active": false } }, { "type": "Microsoft.Web/sites/deployments", "apiVersion": "2018-11-01", "name": "[concat(parameters('FunctionAppName'), '/6b0808e64188414e88d75565027efc59')]", "location": "UK South", "dependsOn": [ "[resourceId('Microsoft.Web/sites', parameters('FunctionAppName'))]" ], "properties": { "status": 4, "author_email": "N/A", "author": "N/A", "deployer": "Push-Deployer", "message": "Created via a push deployment", "start_time": "2020-04-30T16:17:52.965083Z", "end_time": "2020-04-30T16:17:58.0803657Z", "active": false } }, { "type": "Microsoft.Web/sites/deployments", "apiVersion": "2018-11-01", "name": "[concat(parameters('FunctionAppName'), '/9ca87860d72f47049af7ab9bce50ccc1')]", "location": "UK South", "dependsOn": [ "[resourceId('Microsoft.Web/sites', parameters('FunctionAppName'))]" ], "properties": { "status": 4, "author_email": "N/A", "author": "N/A", "deployer": "Push-Deployer", "message": "Created via a push deployment", "start_time": "2020-05-06T13:21:39.9815784Z", "end_time": "2020-05-06T13:21:45.8257955Z", "active": true } }, { "type": "Microsoft.Web/sites/functions", "apiVersion": "2018-11-01", "name": "[concat(parameters('FunctionAppName'), '/CheckPipelineStatus')]", "location": "UK South", "dependsOn": [ "[resourceId('Microsoft.Web/sites', parameters('FunctionAppName'))]" ], "properties": { "script_root_path_href": "https://[parameters('FunctionAppName')].azurewebsites.net/admin/vfs/site/wwwroot/CheckPipelineStatus/", "script_href": "https://[parameters('FunctionAppName')].azurewebsites.net/admin/vfs/site/wwwroot/bin/ADFprocfwk.dll", "config_href": "https://[parameters('FunctionAppName')].azurewebsites.net/admin/vfs/site/wwwroot/CheckPipelineStatus/function.json", "href": "https://[parameters('FunctionAppName')].azurewebsites.net/admin/functions/CheckPipelineStatus", "config": {} } }, { "type": "Microsoft.Web/sites/functions", "apiVersion": "2018-11-01", "name": "[concat(parameters('FunctionAppName'), '/ExecutePipeline')]", "location": "UK South", "dependsOn": [ "[resourceId('Microsoft.Web/sites', parameters('FunctionAppName'))]" ], "properties": { "script_root_path_href": "https://[parameters('FunctionAppName')].azurewebsites.net/admin/vfs/site/wwwroot/ExecutePipeline/", "script_href": "https://[parameters('FunctionAppName')].azurewebsites.net/admin/vfs/site/wwwroot/bin/ADFprocfwk.dll", "config_href": "https://[parameters('FunctionAppName')].azurewebsites.net/admin/vfs/site/wwwroot/ExecutePipeline/function.json", "href": "https://[parameters('FunctionAppName')].azurewebsites.net/admin/functions/ExecutePipeline", "config": {} } }, { "type": "Microsoft.Web/sites/functions", "apiVersion": "2018-11-01", "name": "[concat(parameters('FunctionAppName'), '/GetActivityErrors')]", "location": "UK South", "dependsOn": [ "[resourceId('Microsoft.Web/sites', parameters('FunctionAppName'))]" ], "properties": { "script_root_path_href": "https://[parameters('FunctionAppName')].azurewebsites.net/admin/vfs/site/wwwroot/GetActivityErrors/", "script_href": "https://[parameters('FunctionAppName')].azurewebsites.net/admin/vfs/site/wwwroot/bin/ADFprocfwk.dll", "config_href": "https://[parameters('FunctionAppName')].azurewebsites.net/admin/vfs/site/wwwroot/GetActivityErrors/function.json", "href": "https://[parameters('FunctionAppName')].azurewebsites.net/admin/functions/GetActivityErrors", "config": {} } }, { "type": "Microsoft.Web/sites/functions", "apiVersion": "2018-11-01", "name": "[concat(parameters('FunctionAppName'), '/SendEmail')]", "location": "UK South", "dependsOn": [ "[resourceId('Microsoft.Web/sites', parameters('FunctionAppName'))]" ], "properties": { "script_root_path_href": "https://[parameters('FunctionAppName')].azurewebsites.net/admin/vfs/site/wwwroot/SendEmail/", "script_href": "https://[parameters('FunctionAppName')].azurewebsites.net/admin/vfs/site/wwwroot/bin/ADFprocfwk.dll", "config_href": "https://[parameters('FunctionAppName')].azurewebsites.net/admin/vfs/site/wwwroot/SendEmail/function.json", "href": "https://[parameters('FunctionAppName')].azurewebsites.net/admin/functions/SendEmail", "config": {} } }, { "type": "Microsoft.Web/sites/hostNameBindings", "apiVersion": "2018-11-01", "name": "[concat(parameters('FunctionAppName'), '/', parameters('FunctionAppName'), '.azurewebsites.net')]", "location": "UK South", "dependsOn": [ "[resourceId('Microsoft.Web/sites', parameters('FunctionAppName'))]" ], "properties": { "siteName": "[parameters('FunctionAppName')]", "hostNameType": "Verified" } } ] } ================================================ FILE: ARM Templates/SQL Database/v1.6 Export.json ================================================ { "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { "servers_platformsupport01_name": { "defaultValue": "frameworksupport01", "type": "String" } }, "variables": {}, "resources": [ { "type": "Microsoft.Sql/servers/databases", "apiVersion": "2019-06-01-preview", "name": "[concat(parameters('servers_platformsupport01_name'), '/OrchestrationSupport')]", "location": "uksouth", "sku": { "name": "Standard", "tier": "Standard", "capacity": 20 }, "kind": "v12.0,user", "properties": { "collation": "SQL_Latin1_General_CP1_CI_AS", "maxSizeBytes": 10737418240, "catalogCollation": "SQL_Latin1_General_CP1_CI_AS", "zoneRedundant": false, "readScale": "Disabled", "readReplicaCount": 0, "storageAccountType": "GRS" } }, { "type": "Microsoft.Sql/servers/databases/advisors", "apiVersion": "2014-04-01", "name": "[concat(parameters('servers_platformsupport01_name'), '/OrchestrationSupport/CreateIndex')]", "dependsOn": [ "[resourceId('Microsoft.Sql/servers/databases', parameters('servers_platformsupport01_name'), 'OrchestrationSupport')]" ], "properties": { "autoExecuteValue": "Disabled" } }, { "type": "Microsoft.Sql/servers/databases/advisors", "apiVersion": "2014-04-01", "name": "[concat(parameters('servers_platformsupport01_name'), '/OrchestrationSupport/DbParameterization')]", "dependsOn": [ "[resourceId('Microsoft.Sql/servers/databases', parameters('servers_platformsupport01_name'), 'OrchestrationSupport')]" ], "properties": { "autoExecuteValue": "Disabled" } }, { "type": "Microsoft.Sql/servers/databases/advisors", "apiVersion": "2014-04-01", "name": "[concat(parameters('servers_platformsupport01_name'), '/OrchestrationSupport/DefragmentIndex')]", "dependsOn": [ "[resourceId('Microsoft.Sql/servers/databases', parameters('servers_platformsupport01_name'), 'OrchestrationSupport')]" ], "properties": { "autoExecuteValue": "Disabled" } }, { "type": "Microsoft.Sql/servers/databases/advisors", "apiVersion": "2014-04-01", "name": "[concat(parameters('servers_platformsupport01_name'), '/OrchestrationSupport/DropIndex')]", "dependsOn": [ "[resourceId('Microsoft.Sql/servers/databases', parameters('servers_platformsupport01_name'), 'OrchestrationSupport')]" ], "properties": { "autoExecuteValue": "Disabled" } }, { "type": "Microsoft.Sql/servers/databases/advisors", "apiVersion": "2014-04-01", "name": "[concat(parameters('servers_platformsupport01_name'), '/OrchestrationSupport/ForceLastGoodPlan')]", "dependsOn": [ "[resourceId('Microsoft.Sql/servers/databases', parameters('servers_platformsupport01_name'), 'OrchestrationSupport')]" ], "properties": { "autoExecuteValue": "Enabled" } }, { "type": "Microsoft.Sql/servers/databases/auditingPolicies", "apiVersion": "2014-04-01", "name": "[concat(parameters('servers_platformsupport01_name'), '/OrchestrationSupport/Default')]", "location": "UK South", "dependsOn": [ "[resourceId('Microsoft.Sql/servers/databases', parameters('servers_platformsupport01_name'), 'OrchestrationSupport')]" ], "properties": { "auditingState": "Disabled" } }, { "type": "Microsoft.Sql/servers/databases/auditingSettings", "apiVersion": "2017-03-01-preview", "name": "[concat(parameters('servers_platformsupport01_name'), '/OrchestrationSupport/Default')]", "dependsOn": [ "[resourceId('Microsoft.Sql/servers/databases', parameters('servers_platformsupport01_name'), 'OrchestrationSupport')]" ], "properties": { "state": "Disabled", "retentionDays": 0, "storageAccountSubscriptionId": "00000000-0000-0000-0000-000000000000", "isAzureMonitorTargetEnabled": false } }, { "type": "Microsoft.Sql/servers/databases/backupLongTermRetentionPolicies", "apiVersion": "2017-03-01-preview", "name": "[concat(parameters('servers_platformsupport01_name'), '/OrchestrationSupport/default')]", "dependsOn": [ "[resourceId('Microsoft.Sql/servers/databases', parameters('servers_platformsupport01_name'), 'OrchestrationSupport')]" ], "properties": { "weeklyRetention": "PT0S", "monthlyRetention": "PT0S", "yearlyRetention": "PT0S", "weekOfYear": 0 } }, { "type": "Microsoft.Sql/servers/databases/backupShortTermRetentionPolicies", "apiVersion": "2017-10-01-preview", "name": "[concat(parameters('servers_platformsupport01_name'), '/OrchestrationSupport/default')]", "dependsOn": [ "[resourceId('Microsoft.Sql/servers/databases', parameters('servers_platformsupport01_name'), 'OrchestrationSupport')]" ], "properties": { "retentionDays": 7 } }, { "type": "Microsoft.Sql/servers/databases/extendedAuditingSettings", "apiVersion": "2017-03-01-preview", "name": "[concat(parameters('servers_platformsupport01_name'), '/OrchestrationSupport/Default')]", "dependsOn": [ "[resourceId('Microsoft.Sql/servers/databases', parameters('servers_platformsupport01_name'), 'OrchestrationSupport')]" ], "properties": { "state": "Disabled", "retentionDays": 0, "storageAccountSubscriptionId": "00000000-0000-0000-0000-000000000000", "isAzureMonitorTargetEnabled": false } }, { "type": "Microsoft.Sql/servers/databases/geoBackupPolicies", "apiVersion": "2014-04-01", "name": "[concat(parameters('servers_platformsupport01_name'), '/OrchestrationSupport/Default')]", "location": "UK South", "dependsOn": [ "[resourceId('Microsoft.Sql/servers/databases', parameters('servers_platformsupport01_name'), 'OrchestrationSupport')]" ], "properties": { "state": "Enabled" } }, { "type": "Microsoft.Sql/servers/databases/securityAlertPolicies", "apiVersion": "2018-06-01-preview", "name": "[concat(parameters('servers_platformsupport01_name'), '/OrchestrationSupport/Default')]", "dependsOn": [ "[resourceId('Microsoft.Sql/servers/databases', parameters('servers_platformsupport01_name'), 'OrchestrationSupport')]" ], "properties": { "state": "Disabled", "disabledAlerts": [ "" ], "emailAddresses": [ "" ], "emailAccountAdmins": false, "retentionDays": 0 } }, { "type": "Microsoft.Sql/servers/databases/transparentDataEncryption", "apiVersion": "2014-04-01", "name": "[concat(parameters('servers_platformsupport01_name'), '/OrchestrationSupport/current')]", "location": "UK South", "dependsOn": [ "[resourceId('Microsoft.Sql/servers/databases', parameters('servers_platformsupport01_name'), 'OrchestrationSupport')]" ], "properties": { "status": "Enabled" } }, { "type": "Microsoft.Sql/servers/databases/vulnerabilityAssessments", "apiVersion": "2017-03-01-preview", "name": "[concat(parameters('servers_platformsupport01_name'), '/OrchestrationSupport/Default')]", "dependsOn": [ "[resourceId('Microsoft.Sql/servers/databases', parameters('servers_platformsupport01_name'), 'OrchestrationSupport')]" ], "properties": { "recurringScans": { "isEnabled": false, "emailSubscriptionAdmins": true } } } ] } ================================================ FILE: ARM Templates/Synapse/v2.0 Export.json ================================================ { "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { "workspaceName": { "type": "string", "metadata": "Workspace name", "defaultValue": "" }, "FrameworkFunctions_properties_typeProperties_functionAppUrl": { "type": "string", "defaultValue": "" }, "Keys_properties_typeProperties_baseUrl": { "type": "string", "defaultValue": "" }, "SupportDatabase_properties_typeProperties_connectionString_secretName": { "type": "string", "defaultValue": "" } }, "variables": { "workspaceId": "[concat('Microsoft.Synapse/workspaces/', parameters('workspaceName'))]" }, "resources": [ { "name": "[concat(parameters('workspaceName'), '/01-Grandparent')]", "type": "Microsoft.Synapse/workspaces/pipelines", "apiVersion": "2019-06-01-preview", "properties": { "description": "procfwk grandparent pipeline used optionally to bootstrap any wider processes in your Data Factory that then calls the processing framework.", "activities": [ { "name": "procfwk", "description": "Call procfwk", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "02-Parent", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "BatchName": { "value": "@pipeline().parameters.BatchName", "type": "Expression" } } } } ], "parameters": { "BatchName": { "type": "string", "defaultValue": "NotUsed" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "procfwk", "Grandparent" ] }, "dependsOn": [ "[concat(variables('workspaceId'), '/pipelines/02-Parent')]" ] }, { "name": "[concat(parameters('workspaceName'), '/02-Parent')]", "type": "Microsoft.Synapse/workspaces/pipelines", "apiVersion": "2019-06-01-preview", "properties": { "description": "ADF.procfwk parent pipeline used to bootstrap the orchestration framework in perform the first level ForEach calls in sequence for the metadata stages.", "activities": [ { "name": "Get Stages", "description": "Returns a distinct list of execution stages within the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Set Execution Id", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetStages]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@variables('ExecutionId')", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Stages", "description": "Top level ForEach to sequentially call all processing stages within the framework metadata. Items for iteration passed from the Get Stages lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Stages", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Stages').output.value", "type": "Expression" }, "isSequential": true, "activities": [ { "name": "Stage Executor", "description": "Call to the framework generic child pipeline for a given execution stage.", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Log Stage Preparing", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "03-Child", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "StageId": { "value": "@item().StageId", "type": "Expression" }, "ExecutionId": { "value": "@variables('ExecutionId')", "type": "Expression" } } } }, { "name": "Log Stage Preparing", "description": "Update the current execution table flagging all pipelines within the stage as preparing.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Check and Update Blockers", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogStagePreparing]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@variables('ExecutionId')", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check and Update Blockers", "description": "Used to double check and stop the next execution stage if failures and blockers have be incurred. This also depends on the failure handling property value which defines the stored procedure behaviour.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[CheckForBlockedPipelines]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@variables('ExecutionId')", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Execution Wrapper", "description": "Wrapper to reset and restart processing or create a completely new execution instance of the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Clean Up Previous Run", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[ExecutionWrapper]", "storedProcedureParameters": { "CallingOrchestratorName": { "type": "String", "value": { "value": "@pipeline().DataFactory", "type": "Expression" } }, "BatchName": { "type": "String", "value": { "value": "@pipeline().parameters.BatchName", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Check Outcome and Update Logs", "description": "After a successful execution run the current execution metadata is moved to the long term logging table by this stored procedure call. Otherwise an error will be raised.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Stages", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[UpdateExecutionLog]", "storedProcedureParameters": { "PerformErrorCheck": { "value": { "value": "@bool(1)", "type": "Expression" }, "type": "Boolean" }, "ExecutionId": { "value": { "value": "@variables('ExecutionId')", "type": "Expression" }, "type": "Guid" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check Previous Execution", "description": "Query the current execution table for worker pipelines that require a clean up from the previous execution run.", "type": "Lookup", "dependsOn": [ { "activity": "Execute Precursor", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[CheckPreviousExeuction]", "storedProcedureParameters": { "BatchName": { "type": "String", "value": { "value": "@pipeline().parameters.BatchName", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Clean Up Previous Run", "description": "Handle Worker pipelines that are reported as Running when the parent pipeline is called again. Get what the actual status of those pipelines is.", "type": "ForEach", "dependsOn": [ { "activity": "Check Previous Execution", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Check Metadata Integrity", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Check Previous Execution').output.value", "type": "Expression" }, "isSequential": false, "batchCount": 50, "activities": [ { "name": "Get SPN Details", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": true, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetWorkerAuthDetails]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@item().LocalExecutionId", "type": "Expression" } }, "PipelineId": { "type": "Int32", "value": { "value": "@item().PipelineId", "type": "Expression" } }, "StageId": { "type": "Int32", "value": { "value": "@item().StageId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Pipeline Checking", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineChecking]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Pipeline Status", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Get SPN Details", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Log Pipeline Checking", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "CheckPipelineStatus", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',activity('Get SPN Details').output.firstRow.TenantId,'\",\n \"applicationId\": \"',activity('Get SPN Details').output.firstRow.AppId,'\",\n \"authenticationKey\": \"',activity('Get SPN Details').output.firstRow.AppSecret,'\",\n \"subscriptionId\": \"',activity('Get SPN Details').output.firstRow.SubscriptionId,'\",\n \"resourceGroupName\": \"',item().ResourceGroupName,'\",\n \"orchestratorName\": \"',item().OrchestratorName,'\",\n \"orchestratorType\": \"',item().OrchestratorType,'\",\n \"pipelineName\": \"',item().PipelineName,'\",\n \"runId\": \"',item().PipelineRunId,'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Set Pipeline Status", "description": "Update the metadata depending on the actual pipeline outcome. Using the status as the case.", "type": "Switch", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@activity('Get Pipeline Status').output.ActualStatus", "type": "Expression" }, "cases": [ { "value": "Failed", "activities": [ { "name": "Pipeline Status Failed", "description": "Updates the current execution table with a pipeline status of failed if the function outcome is failed. Also blocks pipelines in the downstream execution stage.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": null, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Succeeded", "activities": [ { "name": "Pipeline Status Succeeded", "description": "Updates the current execution table with a pipeline status of success if the function outcome is succeeded.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Queued", "activities": [ { "name": "Pipeline Status Queued - Running", "description": "Updates the current execution table with a pipeline status of running if the function outcome is queued.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "InProgress", "activities": [ { "name": "Pipeline Status InProgress - Running", "description": "Updates the current execution table with a pipeline status of running if the function outcome is in progress.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Cancelled", "activities": [ { "name": "Pipeline Status Cancelled", "description": "Updates the current execution table with a pipeline status of cancelled if the function outcome is cancelled.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineCancelled]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" }, "CleanUpRun": { "value": { "value": "@bool(1)", "type": "Expression" }, "type": "Boolean" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } ], "defaultActivities": [ { "name": "Pipeline Status Unknown", "description": "Updates the current execution table with a pipeline status of unknown if the function returns an unexpected outcome.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineUnknown]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" }, "CleanUpRun": { "value": { "value": "@bool(1)", "type": "Expression" }, "type": "Boolean" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Set Last Check DateTime", "description": "Update the current execution table with a date time from when the function last checked the pipeline status.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineLastStatusCheck]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Execute Precursor", "description": "Uses the database property value ExecutionPrecursorProc to run any custom logic against the metadata database before the execution run starts.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[ExecutePrecursorProcedure]" }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Set Execution Id", "description": "Set the local execution Id to a pipeline variable for each in several downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Execution Wrapper", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "ExecutionId", "value": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" } } }, { "name": "Check Metadata Integrity", "description": "Performs a series of checks on all metadata held in the framework SQLDB. This is intended to raise errors before an execution run even starts.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Precursor", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[CheckMetadataIntegrity]", "storedProcedureParameters": { "BatchName": { "value": { "value": "@pipeline().parameters.BatchName", "type": "Expression" }, "type": "String" }, "DebugMode": { "value": { "value": "@bool(0)", "type": "Expression" }, "type": "Boolean" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ], "parameters": { "BatchName": { "type": "string", "defaultValue": "NotUsed" } }, "variables": { "ExecutionId": { "type": "String" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "procfwk", "Parent" ] }, "dependsOn": [ "[concat(variables('workspaceId'), '/datasets/GetSetMetadata')]", "[concat(variables('workspaceId'), '/linkedServices/SupportDatabase')]", "[concat(variables('workspaceId'), '/pipelines/03-Child')]", "[concat(variables('workspaceId'), '/linkedServices/FrameworkFunctions')]" ] }, { "name": "[concat(parameters('workspaceName'), '/03-Child')]", "type": "Microsoft.Synapse/workspaces/pipelines", "apiVersion": "2019-06-01-preview", "properties": { "description": "procfwk child pipeline used to execute Worker pipelines within a given execution stage. This pipeline will be called once for each stage, then execute all Workers in parallel.", "activities": [ { "name": "Get Pipelines", "description": "Returns all pipelines from the metadata to be executed within a given processing stage.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelinesInStage]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" } }, "StageId": { "type": "Int32", "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Pipelines", "description": "Second level ForEach to run in parallel all pipelines within the stage. Items for iteration passed from the Get Pipelines lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Pipelines", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Pipelines').output.value", "type": "Expression" }, "isSequential": false, "batchCount": 50, "activities": [ { "name": "Worker Pipeline Executor", "description": "Run the required worker pipeline and wait for its completion. Update metadata once done.", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "04-Infant", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "executionId": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "stageId": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "pipelineId": { "value": "@item().PipelineId", "type": "Expression" } } } } ] } } ], "parameters": { "StageId": { "type": "int" }, "ExecutionId": { "type": "string" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "procfwk", "Child" ] }, "dependsOn": [ "[concat(variables('workspaceId'), '/datasets/GetSetMetadata')]", "[concat(variables('workspaceId'), '/pipelines/04-Infant')]" ] }, { "name": "[concat(parameters('workspaceName'), '/04-Infant')]", "type": "Microsoft.Synapse/workspaces/pipelines", "apiVersion": "2019-06-01-preview", "properties": { "description": "procfwk infant pipeline used to check when the processing pipeline called by the Child completes and passes the resulting status back to the metadata database.", "activities": [ { "name": "Execute Worker Pipeline", "description": "The lowest level executor with the metadata framework to call existing processing pipelines within Data Factory. The function called will block processing and wait for an outcome.", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Log Pipeline Running", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Pipeline Params", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "ExecutePipeline", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',variables('WorkerCoreDetails')[0].tenantId,'\",\n \"applicationId\": \"',variables('WorkerCoreDetails')[0].applicationId,'\",\n \"authenticationKey\": \"',variables('WorkerCoreDetails')[0].authenticationKey,'\",\n \"subscriptionId\": \"',variables('WorkerCoreDetails')[0].subscriptionId,'\",\n \"resourceGroupName\": \"',variables('WorkerCoreDetails')[0].resourceGroupName,'\",\n\t\"orchestratorName\": \"',variables('WorkerCoreDetails')[0].orchestratorName,'\",\n \"orchestratorType\": \"',variables('WorkerCoreDetails')[0].orchestratorType,'\",\n \"pipelineName\": \"',variables('WorkerCoreDetails')[0].pipelineName,'\"',activity('Get Pipeline Params').output.firstRow.Params,'\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Get Pipeline Params", "description": "Returns any parameters from metadata required for the processing pipeline being called. The output can be an empty string if no parameters are required.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelineParameters]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Pipeline Running", "description": "Sets the current pipeline with a status of running within the current execution database table.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Is Target Worker Validate", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Log Execute Function Activity Failure", "description": "Handle true failures from calling out to the Azure Function and update the current execution table accordingly so a restart can occur.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Worker Pipeline", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" }, "CallingActivity": { "value": "ExecuteWorkerPipeline", "type": "String" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Update Run Id", "description": "Provide the actual ADF run ID back to the current execution table for long term logging and alignment between the metadata other Azure monitoring tools.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Set Run Id", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineRunId]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@variables('WorkerRunId')", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check For Alerts", "description": "Checks the properties tables and if any recipients in the database require alerts sending for the current pipeline ID.", "type": "Lookup", "dependsOn": [ { "activity": "Update Run Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Pipeline Result", "dependencyConditions": [ "Completed" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[CheckForEmailAlerts]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Send Alerts", "description": "True = alerts need sending.\nFalse = do nothing.", "type": "IfCondition", "dependsOn": [ { "activity": "Check For Alerts", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@activity('Check For Alerts').output.firstRow.SendAlerts", "type": "Expression" }, "ifTrueActivities": [ { "name": "Get Email Parts", "description": "Return all required content from the metadata database to send an email alerting using the procfwk. The lookup returns the exact content for the function body request.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": true, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetEmailAlertParts]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": true } }, { "name": "Call Email Sender", "description": "Pass off email request to Utils Send Email pipeline.", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Get Email Parts", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "Email Sender", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "Recipients": { "value": "@activity('Get Email Parts').output.firstRow.emailRecipients", "type": "Expression" }, "CcRecipients": { "value": "@activity('Get Email Parts').output.firstRow.emailCcRecipients", "type": "Expression" }, "BccRecipients": { "value": "@activity('Get Email Parts').output.firstRow.emailBccRecipients", "type": "Expression" }, "Subject": { "value": "@activity('Get Email Parts').output.firstRow.emailSubject", "type": "Expression" }, "Body": { "value": "@activity('Get Email Parts').output.firstRow.emailBody", "type": "Expression" }, "Importance": { "value": "@activity('Get Email Parts').output.firstRow.emailImportance", "type": "Expression" } } } } ] } }, { "name": "Wait Until Pipeline Completes", "description": "Loops until the Worker pipeline called completes.\n\nSimple status:\n- Running = new iteration.\n- Done = break.", "type": "Until", "dependsOn": [ { "activity": "Get Wait Duration", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Execute Worker Pipeline", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Run Id", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@variables('WorkerPipelineState')", "type": "Expression" }, "activities": [ { "name": "Get Worker Pipeline Status", "description": "Checks the status of a given processing pipeline and provides the value for the downstream framework activities to act upon.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "CheckPipelineStatus", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',variables('WorkerCoreDetails')[0].tenantId,'\",\n \"applicationId\": \"',variables('WorkerCoreDetails')[0].applicationId,'\",\n \"authenticationKey\": \"',variables('WorkerCoreDetails')[0].authenticationKey,'\",\n \"subscriptionId\": \"',variables('WorkerCoreDetails')[0].subscriptionId,'\",\n \"resourceGroupName\": \"',variables('WorkerCoreDetails')[0].resourceGroupName,'\",\n\t\"orchestratorName\": \"',variables('WorkerCoreDetails')[0].orchestratorName,'\",\n \"orchestratorType\": \"',variables('WorkerCoreDetails')[0].orchestratorType,'\",\n \"pipelineName\": \"',variables('WorkerCoreDetails')[0].pipelineName,'\",\n \"runId\": \"',variables('WorkerRunId'),'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Wait If Running", "description": "True = Do nothing.\nFalse = Wait, before the next iteration.", "type": "IfCondition", "dependsOn": [ { "activity": "Set Worker State", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@variables('WorkerPipelineState')", "type": "Expression" }, "ifFalseActivities": [ { "name": "Wait for Pipeline", "description": "The processing pipeline is still running so Wait before checking its status again.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@activity('Get Wait Duration').output.firstRow.PropertyValue", "type": "Expression" } } } ] } }, { "name": "Set Last Check DateTime", "description": "Update the current execution table with a date time from when the Worker pipeline status was last checked as part of the Until iterations.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Worker Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineLastStatusCheck]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Log Check Function Activity Failure", "description": "Report to the current execution table that the framework pipeline activity has failed. This failure is outside of the scope of the framework and is probably related to a wider platform problem.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Worker Pipeline Status", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "CallingActivity": { "value": "GetWorkerPipelineStatus", "type": "String" }, "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Set Worker State", "description": "Set the bool state of the Worker pipeline to be used by the Until and If expressions. True = Complete, False = Running.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerPipelineState", "value": { "value": "@equals('Complete',activity('Get Worker Pipeline Status').output.SimpleStatus)", "type": "Expression" } } } ], "timeout": "0.00:10:00" } }, { "name": "Set Pipeline Result", "description": "Receives the outcome from the function execution for a given processing pipeline and updates the current execution table with different pipelines status values depending on the result (case).", "type": "Switch", "dependsOn": [ { "activity": "Wait Until Pipeline Completes", "dependencyConditions": [ "Completed" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@activity('Get Worker Pipeline Status').output.ActualStatus", "type": "Expression" }, "cases": [ { "value": "Succeeded", "activities": [ { "name": "Pipeline Status Succeeded", "description": "Updates the current execution table with a pipeline status of success if the function outcome is succeeded.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Failed", "activities": [ { "name": "Pipeline Status Failed", "description": "Updates the current execution table with a pipeline status of failed if the function outcome is failed. Also blocks pipelines in the downstream execution stage.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@variables('WorkerRunId')", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Worker Pipeline Error Details", "description": "Get the activity error details for the run ID of the worker pipeline called. Returns an array of all errors.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "GetActivityErrors", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',variables('WorkerCoreDetails')[0].tenantId,'\",\n \"applicationId\": \"',variables('WorkerCoreDetails')[0].applicationId,'\",\n \"authenticationKey\": \"',variables('WorkerCoreDetails')[0].authenticationKey,'\",\n \"subscriptionId\": \"',variables('WorkerCoreDetails')[0].subscriptionId,'\",\n \"resourceGroupName\": \"',variables('WorkerCoreDetails')[0].resourceGroupName,'\",\n\t\"orchestratorName\": \"',variables('WorkerCoreDetails')[0].orchestratorName,'\",\n \"orchestratorType\": \"',variables('WorkerCoreDetails')[0].orchestratorType,'\",\n \"pipelineName\": \"',variables('WorkerCoreDetails')[0].pipelineName,'\",\n \"runId\": \"',variables('WorkerRunId'),'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Log Error Details", "description": "Parses pipeline error details and persists them to the metadata database error log table.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Worker Pipeline Error Details", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetErrorLogDetails]", "storedProcedureParameters": { "JsonErrorDetails": { "value": { "value": "@string(activity('Get Worker Pipeline Error Details').output)", "type": "Expression" }, "type": "String" }, "LocalExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Cancelled", "activities": [ { "name": "Pipeline Status Cancelled", "description": "Updates the current execution table with a pipeline status of cancelled if the function outcome is cancelled.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineCancelled]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } ], "defaultActivities": [ { "name": "Pipeline Status Unknown", "description": "Updates the current execution table with a pipeline status of unknown if the function returns an unexpected outcome.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineUnknown]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Get Wait Duration", "description": "Return wait duration in seconds from database properties table to be used during each Until iteration when the Worker pipeline is still running.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "PipelineStatusCheckDuration" } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Set Run Id", "description": "Set local variable from activity output once for value reuse in downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Execute Worker Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerRunId", "value": { "value": "@activity('Execute Worker Pipeline').output.RunId", "type": "Expression" } } }, { "name": "Validate Pipeline", "description": "Query the target data factory and establish if the provided worker pipeline name is valid.", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Log Pipeline Validating", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Capture Worker Core Details as an Array", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "ValidatePipeline", "method": "POST", "headers": {}, "body": { "value": "@concat('\n{\n \"tenantId\": \"',variables('WorkerCoreDetails')[0].tenantId,'\",\n \"applicationId\": \"',variables('WorkerCoreDetails')[0].applicationId,'\",\n \"authenticationKey\": \"',variables('WorkerCoreDetails')[0].authenticationKey,'\",\n \"subscriptionId\": \"',variables('WorkerCoreDetails')[0].subscriptionId,'\",\n \"resourceGroupName\": \"',variables('WorkerCoreDetails')[0].resourceGroupName,'\",\n\t\"orchestratorName\": \"',variables('WorkerCoreDetails')[0].orchestratorName,'\",\n \"orchestratorType\": \"',variables('WorkerCoreDetails')[0].orchestratorType,'\",\n \"pipelineName\": \"',variables('WorkerCoreDetails')[0].pipelineName,'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Is Target Worker Validate", "description": "True = the worker pipeline name is valid.\nFalse = the worker pipeline name is invalid. Raise an exception.", "type": "IfCondition", "dependsOn": [ { "activity": "Validate Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@bool(activity('Validate Pipeline').output.PipelineExists)", "type": "Expression" }, "ifFalseActivities": [ { "name": "Throw Exception - Invalid Infant", "description": "Throw an exception with details about the invalid worker pipeline name.", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "Throw Exception", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "Message": { "value": "@concat('Worker pipeline [',variables('WorkerCoreDetails')[0].pipelineName,'] is not valid in target Orchestrator [',variables('WorkerCoreDetails')[0].orchestratorName,']')", "type": "Expression" } } } }, { "name": "Update Execution With Invalid Worker", "description": "Update the current execution table with an informed status for the worker pipeline that couldn't be executed.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "CallingActivity": { "value": "InvalidPipelineName", "type": "String" }, "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Log Validate Function Activity Failure", "description": "Handle true failures from calling out to the Azure Function and update the current execution table accordingly so a restart can occur.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Validate Pipeline", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" }, "CallingActivity": { "value": "ValidatePipeline", "type": "String" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Log Pipeline Validating", "description": "Sets the current pipeline with a status of validating within the current execution database table.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[procfwk].[SetLogPipelineValidating]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Worker Core Details", "description": "Return worker pipeline information for metadata database. Including target data factory, pipeline name and resource group. Return the SPN ID and Secret for the worker pipeline being executed. Called at this level as each pipeline can have a different SPN.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": true, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetWorkerDetailsWrapper]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" } }, "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } }, "StageId": { "type": "Int32", "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Capture Worker Core Details as an Array", "description": "Add all worker pipeline details to a local variable array that can be accessed by each function call requiring the values.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Core Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerCoreDetails", "value": { "value": "@array(activity('Get Worker Core Details').output.firstRow)", "type": "Expression" } } } ], "parameters": { "executionId": { "type": "string" }, "stageId": { "type": "int" }, "pipelineId": { "type": "int" } }, "variables": { "WorkerPipelineState": { "type": "Boolean" }, "WorkerRunId": { "type": "String" }, "WorkerCoreDetails": { "type": "Array" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "procfwk", "Infant" ] }, "dependsOn": [ "[concat(variables('workspaceId'), '/linkedServices/FrameworkFunctions')]", "[concat(variables('workspaceId'), '/datasets/GetSetMetadata')]", "[concat(variables('workspaceId'), '/linkedServices/SupportDatabase')]", "[concat(variables('workspaceId'), '/pipelines/Email Sender')]", "[concat(variables('workspaceId'), '/pipelines/Throw Exception')]" ] }, { "name": "[concat(parameters('workspaceName'), '/Check For Running Pipeline')]", "type": "Microsoft.Synapse/workspaces/pipelines", "apiVersion": "2019-06-01-preview", "properties": { "description": "For a given pipeline and optional batch name establish if a pipeline run is already in progress. Throw an exception if it it.", "activities": [ { "name": "Filter Running Pipelines", "description": "Filter the pipeline runs results for pipelines that exclude the current triggered run and that are currently running (in progress or queued).", "type": "Filter", "dependsOn": [ { "activity": "Switch For Orchestrator Type", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@variables('PipelineRuns')", "type": "Expression" }, "condition": { "value": "@and(not(equals(item().runId,pipeline().parameters.ThisRunId)),or(equals(item().status,'InProgress'),equals(item().status,'Queued')))", "type": "Expression" } } }, { "name": "Get Framework Orchestrator Details", "description": "Using the metadata orchestrators return details about the resource running the framework pipelines.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetFrameworkOrchestratorDetails]", "storedProcedureParameters": { "CallingOrchestratorName": { "type": "String", "value": { "value": "@pipeline().DataFactory", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Get Query Run Days Value", "description": "Using the metadata properties table return the run days value to provide the API request with a date range for pipeline executions.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "PreviousPipelineRunsQueryRange" } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "If Pipeline Is Running", "description": "If the running pipeline count is greater than or equal to one.\nTrue = raise an exception.", "type": "IfCondition", "dependsOn": [ { "activity": "If Using Batch Executions", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@greaterOrEquals(int(variables('RunCount')),1)", "type": "Expression" }, "ifTrueActivities": [ { "name": "Throw Exception - Pipeline Running", "description": "Using the utils pipeline raise an exception to stop the new trigger while a run is already in progress.", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "Throw Exception", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "Message": { "value": "@concat('Provided pipeline name (',pipeline().parameters.PipelineName,') still has a run in progress or queued given the query range parameters set in the properties table.')", "type": "Expression" } } } } ] } }, { "name": "Get Execution Batch Status", "description": "Using the metadata properties table return the flag to indicate if batch execution setting are enabled or disabled.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "UseExecutionBatches" } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "If Using Batch Executions", "description": "True = batch executions are enabled.\nFalse = batch execution are disabled.", "type": "IfCondition", "dependsOn": [ { "activity": "Get Execution Batch Status", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Filter Running Pipelines", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@equals(activity('Get Execution Batch Status').output.firstRow.PropertyValue,string(1))", "type": "Expression" }, "ifFalseActivities": [ { "name": "Set Run Count Without Batch", "description": "Set the pipelines running count variable to be tested later.", "type": "SetVariable", "dependsOn": [], "userProperties": [], "typeProperties": { "variableName": "RunCount", "value": { "value": "@string(activity('Filter Running Pipelines').output.FilteredItemsCount)", "type": "Expression" } } } ], "ifTrueActivities": [ { "name": "Filter for Batch Name", "description": "Further filter the return pipeline runs for any running pipelines with the same batch name value.", "type": "Filter", "dependsOn": [], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Filter Running Pipelines').output.value", "type": "Expression" }, "condition": { "value": "@equals(item().parameters.BatchName,pipeline().parameters.BatchName)", "type": "Expression" } } }, { "name": "Set Run Count for Batch", "description": "Set the resulting pipeline running count variable to be tested later.", "type": "SetVariable", "dependsOn": [ { "activity": "Filter for Batch Name", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "RunCount", "value": { "value": "@string(activity('Filter for Batch Name').output.FilteredItemsCount)", "type": "Expression" } } } ] } }, { "name": "Set Subscription Id", "description": "Set the subscription Id value to a local variable for use in various downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Framework Orchestrator Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "SubscriptionId", "value": { "value": "@activity('Get Framework Orchestrator Details').output.firstRow.SubscriptionId", "type": "Expression" } } }, { "name": "Set Resource Group Name", "description": "Set the resource group name value to a local variable for use in various downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Framework Orchestrator Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "ResourceGroupName", "value": { "value": "@activity('Get Framework Orchestrator Details').output.firstRow.ResourceGroupName", "type": "Expression" } } }, { "name": "Set Orchestrator Type", "description": "Set the orchestrator type value to a local variable for use in various downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Framework Orchestrator Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "OrchestratorType", "value": { "value": "@toUpper(activity('Get Framework Orchestrator Details').output.firstRow.OrchestratorType)", "type": "Expression" } } }, { "name": "Switch For Orchestrator Type", "description": "Switch and handle requests for both Azure Data Factory (ADF) and Azure Synapse Analytics (SYN).", "type": "Switch", "dependsOn": [ { "activity": "Set Orchestrator Type", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Query Run Days", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Resource Group Name", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Subscription Id", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@variables('OrchestratorType')", "type": "Expression" }, "cases": [ { "value": "ADF", "activities": [ { "name": "Check for Valid ADF Pipeline Name", "description": "Use the Azure Management API to return and establish if the framework pipeline exists in the target Data Factory instance, including being deployed.", "type": "WebActivity", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "url": { "value": "https://management.azure.com/subscriptions/@{variables('SubscriptionId')}/resourceGroups/@{variables('ResourceGroupName')}/providers/Microsoft.DataFactory/factories/@{pipeline().DataFactory}/pipelines/@{pipeline().parameters.PipelineName}?api-version=2018-06-01", "type": "Expression" }, "method": "GET", "headers": {}, "authentication": { "type": "MSI", "resource": "https://management.core.windows.net/" } } }, { "name": "Get ADF Pipeline Runs", "description": "Use the Azure Management API to return a list of data factory pipeline runs within the given time window.", "type": "WebActivity", "dependsOn": [ { "activity": "Check for Valid ADF Pipeline Name", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "url": { "value": "https://management.azure.com/subscriptions/@{variables('SubscriptionId')}/resourceGroups/@{variables('ResourceGroupName')}/providers/Microsoft.DataFactory/factories/@{pipeline().DataFactory}/queryPipelineRuns?api-version=2018-06-01", "type": "Expression" }, "method": "POST", "headers": {}, "body": { "value": "{\n \"lastUpdatedAfter\": \"@{adddays(utcnow(),int(variables('QueryRunDays')))}\",\n \"lastUpdatedBefore\": \"@{utcnow()}\",\n \"filters\": [\n {\n \"operand\": \"PipelineName\",\n \"operator\": \"Equals\",\n \"values\": [\n \"@{pipeline().parameters.PipelineName}\"\n ]\n }\n ]\n}", "type": "Expression" }, "authentication": { "type": "MSI", "resource": "https://management.core.windows.net/" } } }, { "name": "Set ADF Runs Output", "description": "Set output to local array for use in downstream filtering and pipeline checks. Use the same array output for both switch cases.", "type": "SetVariable", "dependsOn": [ { "activity": "Get ADF Pipeline Runs", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "PipelineRuns", "value": { "value": "@activity('Get ADF Pipeline Runs').output.value", "type": "Expression" } } } ] }, { "value": "SYN", "activities": [ { "name": "Check for Valid SYN Pipeline Name", "description": "Use the Azure Management API to return and establish if the framework pipeline exists in the target Synapse instance, including being deployed.\n\nSee: https://docs.microsoft.com/en-us/rest/api/synapse/data-plane/pipeline/getpipeline", "type": "WebActivity", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "url": { "value": "https://@{pipeline().DataFactory}.dev.azuresynapse.net/pipelines/@{pipeline().parameters.PipelineName}?api-version=2019-06-01-preview", "type": "Expression" }, "method": "GET", "headers": {}, "authentication": { "type": "MSI", "resource": "https://management.core.windows.net/" } } }, { "name": "Get SYN Pipeline Runs", "description": "Use the Azure Management API to return a list of synapse pipeline runs within the given time window.\n\nSee: https://docs.microsoft.com/en-us/rest/api/synapse/data-plane/pipelinerun/querypipelinerunsbyworkspace", "type": "WebActivity", "dependsOn": [ { "activity": "Check for Valid SYN Pipeline Name", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "url": { "value": "https://@{pipeline().DataFactory}.dev.azuresynapse.net/queryPipelineRuns?api-version=2019-06-01-preview", "type": "Expression" }, "method": "POST", "headers": {}, "body": { "value": "{\n \"lastUpdatedAfter\": \"@{adddays(utcnow(),int(variables('QueryRunDays')))}\",\n \"lastUpdatedBefore\": \"@{utcnow()}\",\n \"filters\": [\n {\n \"operand\": \"PipelineName\",\n \"operator\": \"Equals\",\n \"values\": [\n \"@{pipeline().parameters.PipelineName}\"\n ]\n }\n ]\n}", "type": "Expression" }, "authentication": { "type": "MSI", "resource": "https://management.core.windows.net/" } } }, { "name": "Set SYN Runs Output", "description": "Set output to local array for use in downstream filtering and pipeline checks. Use the same array output for both switch cases.", "type": "SetVariable", "dependsOn": [ { "activity": "Get SYN Pipeline Runs", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "PipelineRuns", "value": { "value": "@activity('Get SYN Pipeline Runs').output.value", "type": "Expression" } } } ] } ], "defaultActivities": [ { "name": "Throw Exception Invalid Orchestrator Type", "description": "Throw exception if switch cases are not met.", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "Throw Exception", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "Message": "Invalid orchestrator type provided. Unable to check pipeline running state." } } } ] } }, { "name": "Set Query Run Days", "description": "Set the query run days value to a local variable for use in various downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Query Run Days Value", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "QueryRunDays", "value": { "value": "@activity('Get Query Run Days Value').output.firstRow.PropertyValue", "type": "Expression" } } } ], "parameters": { "BatchName": { "type": "string", "defaultValue": "NotUsed" }, "PipelineName": { "type": "string" }, "ThisRunId": { "type": "string" } }, "variables": { "SubscriptionId": { "type": "String" }, "RunCount": { "type": "String" }, "ResourceGroupName": { "type": "String" }, "OrchestratorType": { "type": "String" }, "QueryRunDays": { "type": "String" }, "PipelineRuns": { "type": "Array" } }, "folder": { "name": "_ProcFwk/_ProcFwkUtils" }, "annotations": [ "procfwk", "Utils" ] }, "dependsOn": [ "[concat(variables('workspaceId'), '/datasets/GetSetMetadata')]", "[concat(variables('workspaceId'), '/pipelines/Throw Exception')]" ] }, { "name": "[concat(parameters('workspaceName'), '/Email Sender')]", "type": "Microsoft.Synapse/workspaces/pipelines", "apiVersion": "2019-06-01-preview", "properties": { "description": "Provide a simple abstract over the send email function with request body item exposed as pipeline parameters.", "activities": [ { "name": "Send Email", "description": "Use an Azure Function to perform an SMTP client email send operation.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "SendEmail", "method": "POST", "headers": {}, "body": { "value": "{\n\"emailRecipients\": \"@{pipeline().parameters.Recipients}\",\n\"emailCcRecipients\": \"@{pipeline().parameters.CcRecipients}\",\n\"emailBccRecipients\": \"@{pipeline().parameters.BccRecipients}\",\n\"emailSubject\": \"@{pipeline().parameters.Subject}\",\n\"emailBody\": \"@{pipeline().parameters.Body}\",\n\"emailImportance\": \"@{pipeline().parameters.Importance}\"\n}", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } } ], "parameters": { "Recipients": { "type": "string" }, "CcRecipients": { "type": "string" }, "BccRecipients": { "type": "string" }, "Subject": { "type": "string" }, "Body": { "type": "string" }, "Importance": { "type": "string" } }, "folder": { "name": "_ProcFwk/_ProcFwkUtils" }, "annotations": [ "procfwk", "Utils" ] }, "dependsOn": [ "[concat(variables('workspaceId'), '/linkedServices/FrameworkFunctions')]" ] }, { "name": "[concat(parameters('workspaceName'), '/Intentional Error')]", "type": "Microsoft.Synapse/workspaces/pipelines", "apiVersion": "2019-06-01-preview", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait1", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } }, { "name": "Raise Errors or Not", "description": "Framework development worker simulator.", "type": "IfCondition", "dependsOn": [ { "activity": "Wait1", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@equals(pipeline().parameters.RaiseErrors,'true')", "type": "Expression" }, "ifTrueActivities": [ { "name": "Call Fail Procedure", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[dbo].[FailProcedure]", "storedProcedureParameters": { "RaiseError": { "value": { "value": "@pipeline().parameters.RaiseErrors", "type": "Expression" }, "type": "String" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } } ], "parameters": { "RaiseErrors": { "type": "string", "defaultValue": "false" }, "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] }, "dependsOn": [ "[concat(variables('workspaceId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('workspaceName'), '/Throw Exception')]", "type": "Microsoft.Synapse/workspaces/pipelines", "apiVersion": "2019-06-01-preview", "properties": { "description": "Provide a simple way of throwing an exception within Data Factory using TSQL error handling.", "activities": [ { "name": "Raise Error", "description": "Using a SQL database to raise an error/exception but wrapped up as a data factory pipeline. Error message information exposed as a pipeline parameter.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderQuery": { "value": "RAISERROR('@{pipeline().parameters.Message}',16,1);", "type": "Expression" }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } } ], "parameters": { "Message": { "type": "string" } }, "folder": { "name": "_ProcFwk/_ProcFwkUtils" }, "annotations": [ "procfwk", "Utils" ] }, "dependsOn": [ "[concat(variables('workspaceId'), '/datasets/GetSetMetadata')]" ] }, { "name": "[concat(parameters('workspaceName'), '/Wait 1')]", "type": "Microsoft.Synapse/workspaces/pipelines", "apiVersion": "2019-06-01-preview", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait1", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('workspaceName'), '/Wait 10')]", "type": "Microsoft.Synapse/workspaces/pipelines", "apiVersion": "2019-06-01-preview", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait10", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('workspaceName'), '/Wait 2')]", "type": "Microsoft.Synapse/workspaces/pipelines", "apiVersion": "2019-06-01-preview", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait2", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('workspaceName'), '/Wait 3')]", "type": "Microsoft.Synapse/workspaces/pipelines", "apiVersion": "2019-06-01-preview", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait3", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('workspaceName'), '/Wait 4')]", "type": "Microsoft.Synapse/workspaces/pipelines", "apiVersion": "2019-06-01-preview", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait4", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('workspaceName'), '/Wait 5')]", "type": "Microsoft.Synapse/workspaces/pipelines", "apiVersion": "2019-06-01-preview", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait5", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('workspaceName'), '/Wait 6')]", "type": "Microsoft.Synapse/workspaces/pipelines", "apiVersion": "2019-06-01-preview", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait6", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('workspaceName'), '/Wait 7')]", "type": "Microsoft.Synapse/workspaces/pipelines", "apiVersion": "2019-06-01-preview", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait7", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('workspaceName'), '/Wait 8')]", "type": "Microsoft.Synapse/workspaces/pipelines", "apiVersion": "2019-06-01-preview", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait8", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('workspaceName'), '/Wait 9')]", "type": "Microsoft.Synapse/workspaces/pipelines", "apiVersion": "2019-06-01-preview", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait9", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 15 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] }, "dependsOn": [] }, { "name": "[concat(parameters('workspaceName'), '/GetSetMetadata')]", "type": "Microsoft.Synapse/workspaces/datasets", "apiVersion": "2019-06-01-preview", "properties": { "description": "Single generic dataset used to get and set all database metadata.", "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" }, "folder": { "name": "_ProcFwk" }, "annotations": [ "procfwk" ], "type": "AzureSqlTable", "schema": [], "typeProperties": {} }, "dependsOn": [ "[concat(variables('workspaceId'), '/linkedServices/SupportDatabase')]" ] }, { "name": "[concat(parameters('workspaceName'), '/FrameworkFunctions')]", "type": "Microsoft.Synapse/workspaces/linkedServices", "apiVersion": "2019-06-01-preview", "properties": { "description": "Interact with the Azure Functions App used as middle ware when making requests to Worker pipelines. Authentication done at the Function App level.", "annotations": [ "procfwk" ], "type": "AzureFunction", "typeProperties": { "functionAppUrl": "[parameters('FrameworkFunctions_properties_typeProperties_functionAppUrl')]", "functionKey": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "FrameworkFunctionsKey" } } }, "dependsOn": [ "[concat(variables('workspaceId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('workspaceName'), '/Keys')]", "type": "Microsoft.Synapse/workspaces/linkedServices", "apiVersion": "2019-06-01-preview", "properties": { "description": "Connection to Key Vault for all other ADF linked service credentials required to run the processing framework.", "annotations": [ "procfwk" ], "type": "AzureKeyVault", "typeProperties": { "baseUrl": "[parameters('Keys_properties_typeProperties_baseUrl')]" } }, "dependsOn": [] }, { "name": "[concat(parameters('workspaceName'), '/SupportDatabase')]", "type": "Microsoft.Synapse/workspaces/linkedServices", "apiVersion": "2019-06-01-preview", "properties": { "description": "Connection between ADF and processing framework metadata SQLDB.", "annotations": [ "procfwk" ], "type": "AzureSqlDatabase", "typeProperties": { "connectionString": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "[parameters('SupportDatabase_properties_typeProperties_connectionString_secretName')]" } } }, "dependsOn": [ "[concat(variables('workspaceId'), '/linkedServices/Keys')]" ] }, { "name": "[concat(parameters('workspaceName'), '/procfwkforsynapse-WorkspaceDefaultSqlServer')]", "type": "Microsoft.Synapse/workspaces/linkedServices", "apiVersion": "2019-06-01-preview", "properties": { "parameters": { "DBName": { "type": "String" } }, "annotations": [], "type": "AzureSqlDW", "typeProperties": { "connectionString": "[parameters('procfwkforsynapse-WorkspaceDefaultSqlServer_connectionString')]" }, "connectVia": { "referenceName": "AutoResolveIntegrationRuntime", "type": "IntegrationRuntimeReference" } }, "dependsOn": [ "[concat(variables('workspaceId'), '/integrationRuntimes/AutoResolveIntegrationRuntime')]" ] }, { "name": "[concat(parameters('workspaceName'), '/procfwkforsynapse-WorkspaceDefaultStorage')]", "type": "Microsoft.Synapse/workspaces/linkedServices", "apiVersion": "2019-06-01-preview", "properties": { "annotations": [], "type": "AzureBlobFS", "typeProperties": { "url": "[parameters('procfwkforsynapse-WorkspaceDefaultStorage_properties_typeProperties_url')]" }, "connectVia": { "referenceName": "AutoResolveIntegrationRuntime", "type": "IntegrationRuntimeReference" } }, "dependsOn": [ "[concat(variables('workspaceId'), '/integrationRuntimes/AutoResolveIntegrationRuntime')]" ] }, { "name": "[concat(parameters('workspaceName'), '/FunctionalTestingTrigger')]", "type": "Microsoft.Synapse/workspaces/triggers", "apiVersion": "2019-06-01-preview", "properties": { "description": "Used for functional testing of the framework in a dedicated environment.", "annotations": [ "procfwk" ], "runtimeState": "Stopped", "pipelines": [ { "pipelineReference": { "referenceName": "01-Grandparent", "type": "PipelineReference" }, "parameters": {} } ], "type": "ScheduleTrigger", "typeProperties": { "recurrence": { "frequency": "Hour", "interval": 2, "startTime": "2020-04-06T15:00:00Z", "timeZone": "UTC" } } }, "dependsOn": [ "[concat(variables('workspaceId'), '/pipelines/01-Grandparent')]" ] }, { "name": "[concat(parameters('workspaceName'), '/AutoResolveIntegrationRuntime')]", "type": "Microsoft.Synapse/workspaces/integrationRuntimes", "apiVersion": "2019-06-01-preview", "properties": { "type": "Managed", "typeProperties": { "computeProperties": { "location": "AutoResolve", "dataFlowProperties": { "computeType": "General", "coreCount": 8, "timeToLive": 0 } } } }, "dependsOn": [] } ] } ================================================ FILE: CODE_OF_CONDUCT.md ================================================ # Contributor Covenant Code of Conduct ## Our Pledge In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. ## Our Standards Examples of behavior that contributes to creating a positive environment include: * Using welcoming and inclusive language * Being respectful of differing viewpoints and experiences * Gracefully accepting constructive criticism * Focusing on what is best for the community * Showing empathy towards other community members Examples of unacceptable behavior by participants include: * The use of sexualized language or imagery and unwelcome sexual attention or advances * Trolling, insulting/derogatory comments, and personal or political attacks * Public or private harassment * Publishing others' private information, such as a physical or electronic address, without explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting ## Our Responsibilities Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. ## Scope This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at paul@mrpaulandrew.com. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. ## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html [homepage]: https://www.contributor-covenant.org For answers to common questions about this code of conduct, see https://www.contributor-covenant.org/faq ================================================ FILE: CONTRIBUTING.md ================================================ If you'd like to contribute towards this code project please contact me. I kindly request a minimum of 5 years experience working with Azure Data Platform services. Thanks Paul ================================================ FILE: DataFactory/dataset/GetSetMetadata.json ================================================ { "name": "GetSetMetadata", "properties": { "description": "Single generic dataset used to get and set all database metadata.", "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" }, "folder": { "name": "_ProcFwk" }, "annotations": [ "procfwk" ], "type": "AzureSqlTable", "schema": [] } } ================================================ FILE: DataFactory/factory/FrameworkFactory.json ================================================ { "name": "FrameworkFactory", "properties": { "globalConfigurations": { "PipelineBillingEnabled": "true" } }, "location": "uksouth" } ================================================ FILE: DataFactory/integrationRuntime/AzureIR-UKSouth.json ================================================ { "name": "AzureIR-UKSouth", "properties": { "type": "Managed", "typeProperties": { "computeProperties": { "location": "UK South", "dataFlowProperties": { "computeType": "General", "coreCount": 8, "timeToLive": 10, "cleanup": false } } } } } ================================================ FILE: DataFactory/linkedService/FrameworkFunctions.json ================================================ { "name": "FrameworkFunctions", "properties": { "annotations": [ "procfwk" ], "type": "AzureFunction", "typeProperties": { "functionAppUrl": "https://frameworksupportfunctions.azurewebsites.net", "functionKey": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "FrameworkFunctionsKey" } }, "description": "Interact with the Azure Functions App used as middle ware when making requests to Worker pipelines. Authentication done at the Function App level." } } ================================================ FILE: DataFactory/linkedService/Keys.json ================================================ { "name": "Keys", "properties": { "annotations": [ "procfwk" ], "type": "AzureKeyVault", "typeProperties": { "baseUrl": "https://FrameworkKeys.vault.azure.net/" }, "description": "Connection to Key Vault for all other ADF linked service credentials required to run the processing framework." } } ================================================ FILE: DataFactory/linkedService/SupportDatabase.json ================================================ { "name": "SupportDatabase", "properties": { "description": "Connection between ADF and processing framework metadata SQLDB.", "annotations": [ "procfwk" ], "type": "AzureSqlDatabase", "typeProperties": { "connectionString": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "FrameworkMetadataDev" } } } } ================================================ FILE: DataFactory/pipeline/01-Grandparent.json ================================================ { "name": "01-Grandparent", "properties": { "description": "procfwk grandparent pipeline used optionally to bootstrap any wider processes in your Data Factory that then calls the processing framework.", "activities": [ { "name": "procfwk", "description": "Call procfwk", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "02-Parent", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "BatchName": { "value": "@pipeline().parameters.BatchName", "type": "Expression" } } } } ], "parameters": { "BatchName": { "type": "string", "defaultValue": "NotUsed" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "procfwk", "Grandparent" ] } } ================================================ FILE: DataFactory/pipeline/02-Parent.json ================================================ { "name": "02-Parent", "properties": { "description": "ADF.procfwk parent pipeline used to bootstrap the orchestration framework in perform the first level ForEach calls in sequence for the metadata stages.", "activities": [ { "name": "Get Stages", "description": "Returns a distinct list of execution stages within the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Set Execution Id", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[procfwk].[GetStages]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@variables('ExecutionId')", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference" }, "firstRowOnly": false } }, { "name": "Execute Stages", "description": "Top level ForEach to sequentially call all processing stages within the framework metadata. Items for iteration passed from the Get Stages lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Stages", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Stages').output.value", "type": "Expression" }, "isSequential": true, "activities": [ { "name": "Stage Executor", "description": "Call to the framework generic child pipeline for a given execution stage.", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Log Stage Preparing", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "03-Child", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "StageId": { "value": "@item().StageId", "type": "Expression" }, "ExecutionId": { "value": "@variables('ExecutionId')", "type": "Expression" } } } }, { "name": "Log Stage Preparing", "description": "Update the current execution table flagging all pipelines within the stage as preparing.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Check and Update Blockers", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogStagePreparing]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@variables('ExecutionId')", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check and Update Blockers", "description": "Used to double check and stop the next execution stage if failures and blockers have be incurred. This also depends on the failure handling property value which defines the stored procedure behaviour.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[CheckForBlockedPipelines]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@variables('ExecutionId')", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Execution Wrapper", "description": "Wrapper to reset and restart processing or create a completely new execution instance of the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Clean Up Previous Run", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[procfwk].[ExecutionWrapper]", "storedProcedureParameters": { "CallingOrchestratorName": { "type": "String", "value": { "value": "@pipeline().DataFactory", "type": "Expression" } }, "BatchName": { "type": "String", "value": { "value": "@pipeline().parameters.BatchName", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference" } } }, { "name": "Check Outcome and Update Logs", "description": "After a successful execution run the current execution metadata is moved to the long term logging table by this stored procedure call. Otherwise an error will be raised.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Stages", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[UpdateExecutionLog]", "storedProcedureParameters": { "PerformErrorCheck": { "value": { "value": "@bool(1)", "type": "Expression" }, "type": "Boolean" }, "ExecutionId": { "value": { "value": "@variables('ExecutionId')", "type": "Expression" }, "type": "Guid" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check Previous Execution", "description": "Query the current execution table for worker pipelines that require a clean up from the previous execution run.", "type": "Lookup", "dependsOn": [ { "activity": "Execute Precursor", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[procfwk].[CheckPreviousExeuction]", "storedProcedureParameters": { "BatchName": { "type": "String", "value": { "value": "@pipeline().parameters.BatchName", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference" }, "firstRowOnly": false } }, { "name": "Clean Up Previous Run", "description": "Handle Worker pipelines that are reported as Running when the parent pipeline is called again. Get what the actual status of those pipelines is.", "type": "ForEach", "dependsOn": [ { "activity": "Check Previous Execution", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Check Metadata Integrity", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Check Previous Execution').output.value", "type": "Expression" }, "isSequential": false, "batchCount": 50, "activities": [ { "name": "Get SPN Details", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": true, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[procfwk].[GetWorkerAuthDetails]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@item().LocalExecutionId", "type": "Expression" } }, "PipelineId": { "type": "Int32", "value": { "value": "@item().PipelineId", "type": "Expression" } }, "StageId": { "type": "Int32", "value": { "value": "@item().StageId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference" } } }, { "name": "Log Pipeline Checking", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineChecking]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Pipeline Status", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Get SPN Details", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Log Pipeline Checking", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "CheckPipelineStatus", "method": "POST", "body": { "value": "@concat('\n{\n \"tenantId\": \"',activity('Get SPN Details').output.firstRow.TenantId,'\",\n \"applicationId\": \"',activity('Get SPN Details').output.firstRow.AppId,'\",\n \"authenticationKey\": \"',activity('Get SPN Details').output.firstRow.AppSecret,'\",\n \"subscriptionId\": \"',activity('Get SPN Details').output.firstRow.SubscriptionId,'\",\n \"resourceGroupName\": \"',item().ResourceGroupName,'\",\n \"orchestratorName\": \"',item().OrchestratorName,'\",\n \"orchestratorType\": \"',item().OrchestratorType,'\",\n \"pipelineName\": \"',item().PipelineName,'\",\n \"runId\": \"',item().PipelineRunId,'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Set Pipeline Status", "description": "Update the metadata depending on the actual pipeline outcome. Using the status as the case.", "type": "Switch", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@activity('Get Pipeline Status').output.ActualStatus", "type": "Expression" }, "cases": [ { "value": "Failed", "activities": [ { "name": "Pipeline Status Failed", "description": "Updates the current execution table with a pipeline status of failed if the function outcome is failed. Also blocks pipelines in the downstream execution stage.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": null, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Succeeded", "activities": [ { "name": "Pipeline Status Succeeded", "description": "Updates the current execution table with a pipeline status of success if the function outcome is succeeded.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineSuccess]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Queued", "activities": [ { "name": "Pipeline Status Queued - Running", "description": "Updates the current execution table with a pipeline status of running if the function outcome is queued.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "InProgress", "activities": [ { "name": "Pipeline Status InProgress - Running", "description": "Updates the current execution table with a pipeline status of running if the function outcome is in progress.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Cancelled", "activities": [ { "name": "Pipeline Status Cancelled", "description": "Updates the current execution table with a pipeline status of cancelled if the function outcome is cancelled.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineCancelled]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" }, "CleanUpRun": { "value": { "value": "@bool(1)", "type": "Expression" }, "type": "Boolean" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } ], "defaultActivities": [ { "name": "Pipeline Status Unknown", "description": "Updates the current execution table with a pipeline status of unknown if the function returns an unexpected outcome.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineUnknown]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" }, "CleanUpRun": { "value": { "value": "@bool(1)", "type": "Expression" }, "type": "Boolean" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Set Last Check DateTime", "description": "Update the current execution table with a date time from when the function last checked the pipeline status.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineLastStatusCheck]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Execute Precursor", "description": "Uses the database property value ExecutionPrecursorProc to run any custom logic against the metadata database before the execution run starts.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Is Parent Already Running", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[ExecutePrecursorProcedure]" }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Set Execution Id", "description": "Set the local execution Id to a pipeline variable for each in several downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Execution Wrapper", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "ExecutionId", "value": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" } } }, { "name": "Is Parent Already Running", "description": "Establish before anything else if the parent pipeline is already running. Batch execution aware.", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "Check For Running Pipeline", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "BatchName": { "value": "@pipeline().parameters.BatchName", "type": "Expression" }, "PipelineName": { "value": "@pipeline().Pipeline", "type": "Expression" }, "ThisRunId": { "value": "@pipeline().RunId", "type": "Expression" } } } }, { "name": "Check Metadata Integrity", "description": "Performs a series of checks on all metadata held in the framework SQLDB. This is intended to raise errors before an execution run even starts.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Precursor", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[CheckMetadataIntegrity]", "storedProcedureParameters": { "BatchName": { "value": { "value": "@pipeline().parameters.BatchName", "type": "Expression" }, "type": "String" }, "DebugMode": { "value": { "value": "@bool(0)", "type": "Expression" }, "type": "Boolean" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ], "parameters": { "BatchName": { "type": "string", "defaultValue": "NotUsed" } }, "variables": { "ExecutionId": { "type": "String" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "procfwk", "Parent" ] } } ================================================ FILE: DataFactory/pipeline/03-Child.json ================================================ { "name": "03-Child", "properties": { "description": "procfwk child pipeline used to execute Worker pipelines within a given execution stage. This pipeline will be called once for each stage, then execute all Workers in parallel.", "activities": [ { "name": "Get Pipelines", "description": "Returns all pipelines from the metadata to be executed within a given processing stage.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[procfwk].[GetPipelinesInStage]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" } }, "StageId": { "type": "Int32", "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference" }, "firstRowOnly": false } }, { "name": "Execute Pipelines", "description": "Second level ForEach to run in parallel all pipelines within the stage. Items for iteration passed from the Get Pipelines lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Pipelines", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Pipelines').output.value", "type": "Expression" }, "isSequential": false, "batchCount": 50, "activities": [ { "name": "Worker Pipeline Executor", "description": "Run the required worker pipeline and wait for its completion. Update metadata once done.", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "04-Infant", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "executionId": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "stageId": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "pipelineId": { "value": "@item().PipelineId", "type": "Expression" } } } } ] } } ], "parameters": { "StageId": { "type": "int" }, "ExecutionId": { "type": "string" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "procfwk", "Child" ] } } ================================================ FILE: DataFactory/pipeline/04-Infant.json ================================================ { "name": "04-Infant", "properties": { "description": "procfwk infant pipeline used to check when the processing pipeline called by the Child completes and passes the resulting status back to the metadata database.", "activities": [ { "name": "Execute Worker Pipeline", "description": "The lowest level executor with the metadata framework to call existing processing pipelines within Data Factory. The function called will block processing and wait for an outcome.", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Log Pipeline Running", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Pipeline Params", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "ExecutePipeline", "method": "POST", "body": { "value": "@concat('\n{\n \"tenantId\": \"',variables('WorkerCoreDetails')[0].tenantId,'\",\n \"applicationId\": \"',variables('WorkerCoreDetails')[0].applicationId,'\",\n \"authenticationKey\": \"',variables('WorkerCoreDetails')[0].authenticationKey,'\",\n \"subscriptionId\": \"',variables('WorkerCoreDetails')[0].subscriptionId,'\",\n \"resourceGroupName\": \"',variables('WorkerCoreDetails')[0].resourceGroupName,'\",\n\t\"orchestratorName\": \"',variables('WorkerCoreDetails')[0].orchestratorName,'\",\n \"orchestratorType\": \"',variables('WorkerCoreDetails')[0].orchestratorType,'\",\n \"pipelineName\": \"',variables('WorkerCoreDetails')[0].pipelineName,'\"',activity('Get Pipeline Params').output.firstRow.Params,'\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Get Pipeline Params", "description": "Returns any parameters from metadata required for the processing pipeline being called. The output can be an empty string if no parameters are required.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[procfwk].[GetPipelineParameters]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference" } } }, { "name": "Log Pipeline Running", "description": "Sets the current pipeline with a status of running within the current execution database table.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Is Target Worker Validate", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Log Execute Function Activity Failure", "description": "Handle true failures from calling out to the Azure Function and update the current execution table accordingly so a restart can occur.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Worker Pipeline", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" }, "CallingActivity": { "value": "ExecuteWorkerPipeline", "type": "String" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Update Run Id", "description": "Provide the actual ADF run ID back to the current execution table for long term logging and alignment between the metadata other Azure monitoring tools.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Set Run Id", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineRunId]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@variables('WorkerRunId')", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check For Alerts", "description": "Checks the properties tables and if any recipients in the database require alerts sending for the current pipeline ID.", "type": "Lookup", "dependsOn": [ { "activity": "Update Run Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Pipeline Result", "dependencyConditions": [ "Completed" ] } ], "policy": { "timeout": "0.00:00:30", "retry": 3, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[procfwk].[CheckForEmailAlerts]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference" }, "firstRowOnly": true } }, { "name": "Send Alerts", "description": "True = alerts need sending.\nFalse = do nothing.", "type": "IfCondition", "dependsOn": [ { "activity": "Check For Alerts", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@activity('Check For Alerts').output.firstRow.SendAlerts", "type": "Expression" }, "ifTrueActivities": [ { "name": "Get Email Parts", "description": "Return all required content from the metadata database to send an email alerting using the procfwk. The lookup returns the exact content for the function body request.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": true, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[procfwk].[GetEmailAlertParts]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference" }, "firstRowOnly": true } }, { "name": "Call Email Sender", "description": "Pass off email request to Utils Send Email pipeline.", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Get Email Parts", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "Email Sender", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "Recipients": { "value": "@activity('Get Email Parts').output.firstRow.emailRecipients", "type": "Expression" }, "CcRecipients": { "value": "@activity('Get Email Parts').output.firstRow.emailCcRecipients", "type": "Expression" }, "BccRecipients": { "value": "@activity('Get Email Parts').output.firstRow.emailBccRecipients", "type": "Expression" }, "Subject": { "value": "@activity('Get Email Parts').output.firstRow.emailSubject", "type": "Expression" }, "Body": { "value": "@activity('Get Email Parts').output.firstRow.emailBody", "type": "Expression" }, "Importance": { "value": "@activity('Get Email Parts').output.firstRow.emailImportance", "type": "Expression" } } } } ] } }, { "name": "Wait Until Pipeline Completes", "description": "Loops until the Worker pipeline called completes.\n\nSimple status:\n- Running = new iteration.\n- Done = break.", "type": "Until", "dependsOn": [ { "activity": "Get Wait Duration", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Execute Worker Pipeline", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Run Id", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@variables('WorkerPipelineState')", "type": "Expression" }, "activities": [ { "name": "Get Worker Pipeline Status", "description": "Checks the status of a given processing pipeline and provides the value for the downstream framework activities to act upon.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "0.03:59:59", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "CheckPipelineStatus", "method": "POST", "body": { "value": "@concat('\n{\n \"tenantId\": \"',variables('WorkerCoreDetails')[0].tenantId,'\",\n \"applicationId\": \"',variables('WorkerCoreDetails')[0].applicationId,'\",\n \"authenticationKey\": \"',variables('WorkerCoreDetails')[0].authenticationKey,'\",\n \"subscriptionId\": \"',variables('WorkerCoreDetails')[0].subscriptionId,'\",\n \"resourceGroupName\": \"',variables('WorkerCoreDetails')[0].resourceGroupName,'\",\n\t\"orchestratorName\": \"',variables('WorkerCoreDetails')[0].orchestratorName,'\",\n \"orchestratorType\": \"',variables('WorkerCoreDetails')[0].orchestratorType,'\",\n \"pipelineName\": \"',variables('WorkerCoreDetails')[0].pipelineName,'\",\n \"runId\": \"',variables('WorkerRunId'),'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Wait If Running", "description": "True = Do nothing.\nFalse = Wait, before the next iteration.", "type": "IfCondition", "dependsOn": [ { "activity": "Set Worker State", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@variables('WorkerPipelineState')", "type": "Expression" }, "ifFalseActivities": [ { "name": "Wait for Pipeline", "description": "The processing pipeline is still running so Wait before checking its status again.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@activity('Get Wait Duration').output.firstRow.PropertyValue", "type": "Expression" } } } ] } }, { "name": "Set Last Check DateTime", "description": "Update the current execution table with a date time from when the Worker pipeline status was last checked as part of the Until iterations.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Worker Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineLastStatusCheck]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Log Check Function Activity Failure", "description": "Report to the current execution table that the framework pipeline activity has failed. This failure is outside of the scope of the framework and is probably related to a wider platform problem.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Worker Pipeline Status", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "CallingActivity": { "value": "GetWorkerPipelineStatus", "type": "String" }, "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Set Worker State", "description": "Set the bool state of the Worker pipeline to be used by the Until and If expressions. True = Complete, False = Running.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerPipelineState", "value": { "value": "@equals('Complete',activity('Get Worker Pipeline Status').output.SimpleStatus)", "type": "Expression" } } } ], "timeout": "0.04:00:00" } }, { "name": "Set Pipeline Result", "description": "Receives the outcome from the function execution for a given processing pipeline and updates the current execution table with different pipelines status values depending on the result (case).", "type": "Switch", "dependsOn": [ { "activity": "Wait Until Pipeline Completes", "dependencyConditions": [ "Completed" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@activity('Get Worker Pipeline Status').output.ActualStatus", "type": "Expression" }, "cases": [ { "value": "Succeeded", "activities": [ { "name": "Pipeline Status Succeeded", "description": "Updates the current execution table with a pipeline status of success if the function outcome is succeeded.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:01:00", "retry": 2, "retryIntervalInSeconds": 5, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineSuccess]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Failed", "activities": [ { "name": "Pipeline Status Failed", "description": "Updates the current execution table with a pipeline status of failed if the function outcome is failed. Also blocks pipelines in the downstream execution stage.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:01:00", "retry": 2, "retryIntervalInSeconds": 5, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@variables('WorkerRunId')", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Worker Pipeline Error Details", "description": "Get the activity error details for the run ID of the worker pipeline called. Returns an array of all errors.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "GetActivityErrors", "method": "POST", "body": { "value": "@concat('\n{\n \"tenantId\": \"',variables('WorkerCoreDetails')[0].tenantId,'\",\n \"applicationId\": \"',variables('WorkerCoreDetails')[0].applicationId,'\",\n \"authenticationKey\": \"',variables('WorkerCoreDetails')[0].authenticationKey,'\",\n \"subscriptionId\": \"',variables('WorkerCoreDetails')[0].subscriptionId,'\",\n \"resourceGroupName\": \"',variables('WorkerCoreDetails')[0].resourceGroupName,'\",\n\t\"orchestratorName\": \"',variables('WorkerCoreDetails')[0].orchestratorName,'\",\n \"orchestratorType\": \"',variables('WorkerCoreDetails')[0].orchestratorType,'\",\n \"pipelineName\": \"',variables('WorkerCoreDetails')[0].pipelineName,'\",\n \"runId\": \"',variables('WorkerRunId'),'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Log Error Details", "description": "Parses pipeline error details and persists them to the metadata database error log table.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Worker Pipeline Error Details", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:01:00", "retry": 2, "retryIntervalInSeconds": 5, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetErrorLogDetails]", "storedProcedureParameters": { "JsonErrorDetails": { "value": { "value": "@string(activity('Get Worker Pipeline Error Details').output)", "type": "Expression" }, "type": "String" }, "LocalExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Cancelled", "activities": [ { "name": "Pipeline Status Cancelled", "description": "Updates the current execution table with a pipeline status of cancelled if the function outcome is cancelled.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:01:00", "retry": 2, "retryIntervalInSeconds": 5, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineCancelled]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } ], "defaultActivities": [ { "name": "Pipeline Status Unknown", "description": "Updates the current execution table with a pipeline status of unknown if the function returns an unexpected outcome.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:01:00", "retry": 2, "retryIntervalInSeconds": 5, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineUnknown]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Get Wait Duration", "description": "Return wait duration in seconds from database properties table to be used during each Until iteration when the Worker pipeline is still running.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "PipelineStatusCheckDuration" } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference" } } }, { "name": "Set Run Id", "description": "Set local variable from activity output once for value reuse in downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Execute Worker Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerRunId", "value": { "value": "@activity('Execute Worker Pipeline').output.RunId", "type": "Expression" } } }, { "name": "Validate Pipeline", "description": "Query the target data factory and establish if the provided worker pipeline name is valid.", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Log Pipeline Validating", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Capture Worker Core Details as an Array", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "ValidatePipeline", "method": "POST", "body": { "value": "@concat('\n{\n \"tenantId\": \"',variables('WorkerCoreDetails')[0].tenantId,'\",\n \"applicationId\": \"',variables('WorkerCoreDetails')[0].applicationId,'\",\n \"authenticationKey\": \"',variables('WorkerCoreDetails')[0].authenticationKey,'\",\n \"subscriptionId\": \"',variables('WorkerCoreDetails')[0].subscriptionId,'\",\n \"resourceGroupName\": \"',variables('WorkerCoreDetails')[0].resourceGroupName,'\",\n\t\"orchestratorName\": \"',variables('WorkerCoreDetails')[0].orchestratorName,'\",\n \"orchestratorType\": \"',variables('WorkerCoreDetails')[0].orchestratorType,'\",\n \"pipelineName\": \"',variables('WorkerCoreDetails')[0].pipelineName,'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Is Target Worker Validate", "description": "True = the worker pipeline name is valid.\nFalse = the worker pipeline name is invalid. Raise an exception.", "type": "IfCondition", "dependsOn": [ { "activity": "Validate Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@bool(activity('Validate Pipeline').output.PipelineExists)", "type": "Expression" }, "ifFalseActivities": [ { "name": "Throw Exception - Invalid Infant", "description": "Throw an exception with details about the invalid worker pipeline name.", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "Throw Exception", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "Message": { "value": "@concat('Worker pipeline [',variables('WorkerCoreDetails')[0].pipelineName,'] is not valid in target Orchestrator [',variables('WorkerCoreDetails')[0].orchestratorName,']')", "type": "Expression" } } } }, { "name": "Update Execution With Invalid Worker", "description": "Update the current execution table with an informed status for the worker pipeline that couldn't be executed.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "CallingActivity": { "value": "InvalidPipelineName", "type": "String" }, "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Log Validate Function Activity Failure", "description": "Handle true failures from calling out to the Azure Function and update the current execution table accordingly so a restart can occur.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Validate Pipeline", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" }, "CallingActivity": { "value": "ValidatePipeline", "type": "String" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Log Pipeline Validating", "description": "Sets the current pipeline with a status of validating within the current execution database table.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineValidating]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Worker Core Details", "description": "Return worker pipeline information for metadata database. Including target data factory, pipeline name and resource group. Return the SPN ID and Secret for the worker pipeline being executed. Called at this level as each pipeline can have a different SPN.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": true, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[procfwk].[GetWorkerDetailsWrapper]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" } }, "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } }, "StageId": { "type": "Int32", "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference" } } }, { "name": "Capture Worker Core Details as an Array", "description": "Add all worker pipeline details to a local variable array that can be accessed by each function call requiring the values.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Core Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerCoreDetails", "value": { "value": "@array(activity('Get Worker Core Details').output.firstRow)", "type": "Expression" } } } ], "parameters": { "executionId": { "type": "string" }, "stageId": { "type": "int" }, "pipelineId": { "type": "int" } }, "variables": { "WorkerPipelineState": { "type": "Boolean" }, "WorkerRunId": { "type": "String" }, "WorkerCoreDetails": { "type": "Array" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "procfwk", "Infant" ] } } ================================================ FILE: DataFactory/pipeline/Check For Running Pipeline.json ================================================ { "name": "Check For Running Pipeline", "properties": { "description": "For a given pipeline and optional batch name establish if a pipeline run is already in progress. Throw an exception if it it.", "activities": [ { "name": "Filter Running Pipelines", "description": "Filter the pipeline runs results for pipelines that exclude the current triggered run and that are currently running (in progress or queued).", "type": "Filter", "dependsOn": [ { "activity": "Switch For Orchestrator Type", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@variables('PipelineRuns')", "type": "Expression" }, "condition": { "value": "@and(not(equals(item().runId,pipeline().parameters.ThisRunId)),or(equals(item().status,'InProgress'),equals(item().status,'Queued')))", "type": "Expression" } } }, { "name": "Get Framework Orchestrator Details", "description": "Using the metadata orchestrators return details about the resource running the framework pipelines.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[procfwk].[GetFrameworkOrchestratorDetails]", "storedProcedureParameters": { "CallingOrchestratorName": { "type": "String", "value": { "value": "@pipeline().DataFactory", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference" } } }, { "name": "Get Query Run Days Value", "description": "Using the metadata properties table return the run days value to provide the API request with a date range for pipeline executions.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "PreviousPipelineRunsQueryRange" } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference" } } }, { "name": "If Pipeline Is Running", "description": "If the running pipeline count is greater than or equal to one.\nTrue = raise an exception.", "type": "IfCondition", "dependsOn": [ { "activity": "If Using Batch Executions", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@greaterOrEquals(int(variables('RunCount')),1)", "type": "Expression" }, "ifTrueActivities": [ { "name": "Throw Exception - Pipeline Running", "description": "Using the utils pipeline raise an exception to stop the new trigger while a run is already in progress.", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "Throw Exception", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "Message": { "value": "@concat('Provided pipeline name (',pipeline().parameters.PipelineName,') still has a run in progress or queued given the query range parameters set in the properties table.')", "type": "Expression" } } } } ] } }, { "name": "Get Execution Batch Status", "description": "Using the metadata properties table return the flag to indicate if batch execution setting are enabled or disabled.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "UseExecutionBatches" } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference" } } }, { "name": "If Using Batch Executions", "description": "True = batch executions are enabled.\nFalse = batch execution are disabled.", "type": "IfCondition", "dependsOn": [ { "activity": "Get Execution Batch Status", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Filter Running Pipelines", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@equals(activity('Get Execution Batch Status').output.firstRow.PropertyValue,string(1))", "type": "Expression" }, "ifFalseActivities": [ { "name": "Set Run Count Without Batch", "description": "Set the pipelines running count variable to be tested later.", "type": "SetVariable", "dependsOn": [], "userProperties": [], "typeProperties": { "variableName": "RunCount", "value": { "value": "@string(activity('Filter Running Pipelines').output.FilteredItemsCount)", "type": "Expression" } } } ], "ifTrueActivities": [ { "name": "Filter for Batch Name", "description": "Further filter the return pipeline runs for any running pipelines with the same batch name value.", "type": "Filter", "dependsOn": [], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Filter Running Pipelines').output.value", "type": "Expression" }, "condition": { "value": "@equals(item().parameters.BatchName,pipeline().parameters.BatchName)", "type": "Expression" } } }, { "name": "Set Run Count for Batch", "description": "Set the resulting pipeline running count variable to be tested later.", "type": "SetVariable", "dependsOn": [ { "activity": "Filter for Batch Name", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "RunCount", "value": { "value": "@string(activity('Filter for Batch Name').output.FilteredItemsCount)", "type": "Expression" } } } ] } }, { "name": "Set Subscription Id", "description": "Set the subscription Id value to a local variable for use in various downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Framework Orchestrator Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "SubscriptionId", "value": { "value": "@activity('Get Framework Orchestrator Details').output.firstRow.SubscriptionId", "type": "Expression" } } }, { "name": "Set Resource Group Name", "description": "Set the resource group name value to a local variable for use in various downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Framework Orchestrator Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "ResourceGroupName", "value": { "value": "@activity('Get Framework Orchestrator Details').output.firstRow.ResourceGroupName", "type": "Expression" } } }, { "name": "Set Orchestrator Type", "description": "Set the orchestrator type value to a local variable for use in various downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Framework Orchestrator Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "OrchestratorType", "value": { "value": "@toUpper(activity('Get Framework Orchestrator Details').output.firstRow.OrchestratorType)", "type": "Expression" } } }, { "name": "Switch For Orchestrator Type", "description": "Switch and handle requests for both Azure Data Factory (ADF) and Azure Synapse Analytics (SYN).", "type": "Switch", "dependsOn": [ { "activity": "Set Orchestrator Type", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Query Run Days", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Resource Group Name", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Subscription Id", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@variables('OrchestratorType')", "type": "Expression" }, "cases": [ { "value": "ADF", "activities": [ { "name": "Check for Valid ADF Pipeline Name", "description": "Use the Azure Management API to return and establish if the framework pipeline exists in the target Data Factory instance, including being deployed.", "type": "WebActivity", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "url": { "value": "https://management.azure.com/subscriptions/@{variables('SubscriptionId')}/resourceGroups/@{variables('ResourceGroupName')}/providers/Microsoft.DataFactory/factories/@{pipeline().DataFactory}/pipelines/@{pipeline().parameters.PipelineName}?api-version=2018-06-01", "type": "Expression" }, "method": "GET", "authentication": { "type": "MSI", "resource": "https://management.core.windows.net/" } } }, { "name": "Get ADF Pipeline Runs", "description": "Use the Azure Management API to return a list of data factory pipeline runs within the given time window.", "type": "WebActivity", "dependsOn": [ { "activity": "Check for Valid ADF Pipeline Name", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "url": { "value": "https://management.azure.com/subscriptions/@{variables('SubscriptionId')}/resourceGroups/@{variables('ResourceGroupName')}/providers/Microsoft.DataFactory/factories/@{pipeline().DataFactory}/queryPipelineRuns?api-version=2018-06-01", "type": "Expression" }, "method": "POST", "body": { "value": "{\n \"lastUpdatedAfter\": \"@{adddays(utcnow(),int(variables('QueryRunDays')))}\",\n \"lastUpdatedBefore\": \"@{utcnow()}\",\n \"filters\": [\n {\n \"operand\": \"PipelineName\",\n \"operator\": \"Equals\",\n \"values\": [\n \"@{pipeline().parameters.PipelineName}\"\n ]\n }\n ]\n}", "type": "Expression" }, "authentication": { "type": "MSI", "resource": "https://management.core.windows.net/" } } }, { "name": "Set ADF Runs Output", "description": "Set output to local array for use in downstream filtering and pipeline checks. Use the same array output for both switch cases.", "type": "SetVariable", "dependsOn": [ { "activity": "Get ADF Pipeline Runs", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "PipelineRuns", "value": { "value": "@activity('Get ADF Pipeline Runs').output.value", "type": "Expression" } } } ] }, { "value": "SYN", "activities": [ { "name": "Check for Valid SYN Pipeline Name", "description": "Use the Azure Management API to return and establish if the framework pipeline exists in the target Synapse instance, including being deployed.\n\nSee: https://docs.microsoft.com/en-us/rest/api/synapse/data-plane/pipeline/getpipeline", "type": "WebActivity", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "url": { "value": "https://@{pipeline().DataFactory}.dev.azuresynapse.net/pipelines/@{pipeline().parameters.PipelineName}?api-version=2019-06-01-preview", "type": "Expression" }, "method": "GET", "authentication": { "type": "MSI", "resource": "https://management.core.windows.net/" } } }, { "name": "Get SYN Pipeline Runs", "description": "Use the Azure Management API to return a list of synapse pipeline runs within the given time window.\n\nSee: https://docs.microsoft.com/en-us/rest/api/synapse/data-plane/pipelinerun/querypipelinerunsbyworkspace", "type": "WebActivity", "dependsOn": [ { "activity": "Check for Valid SYN Pipeline Name", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "url": { "value": "https://@{pipeline().DataFactory}.dev.azuresynapse.net/queryPipelineRuns?api-version=2019-06-01-preview", "type": "Expression" }, "method": "POST", "body": { "value": "{\n \"lastUpdatedAfter\": \"@{adddays(utcnow(),int(variables('QueryRunDays')))}\",\n \"lastUpdatedBefore\": \"@{utcnow()}\",\n \"filters\": [\n {\n \"operand\": \"PipelineName\",\n \"operator\": \"Equals\",\n \"values\": [\n \"@{pipeline().parameters.PipelineName}\"\n ]\n }\n ]\n}", "type": "Expression" }, "authentication": { "type": "MSI", "resource": "https://management.core.windows.net/" } } }, { "name": "Set SYN Runs Output", "description": "Set output to local array for use in downstream filtering and pipeline checks. Use the same array output for both switch cases.", "type": "SetVariable", "dependsOn": [ { "activity": "Get SYN Pipeline Runs", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "PipelineRuns", "value": { "value": "@activity('Get SYN Pipeline Runs').output.value", "type": "Expression" } } } ] } ], "defaultActivities": [ { "name": "Throw Exception Invalid Orchestrator Type", "description": "Throw exception if switch cases are not met.", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "Throw Exception", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "Message": "Invalid orchestrator type provided. Unable to check pipeline running state." } } } ] } }, { "name": "Set Query Run Days", "description": "Set the query run days value to a local variable for use in various downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Query Run Days Value", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "QueryRunDays", "value": { "value": "@activity('Get Query Run Days Value').output.firstRow.PropertyValue", "type": "Expression" } } } ], "parameters": { "BatchName": { "type": "string", "defaultValue": "NotUsed" }, "PipelineName": { "type": "string" }, "ThisRunId": { "type": "string" } }, "variables": { "SubscriptionId": { "type": "String" }, "RunCount": { "type": "String" }, "ResourceGroupName": { "type": "String" }, "OrchestratorType": { "type": "String" }, "QueryRunDays": { "type": "String" }, "PipelineRuns": { "type": "Array" } }, "folder": { "name": "_ProcFwk/_ProcFwkUtils" }, "annotations": [ "procfwk", "Utils" ] } } ================================================ FILE: DataFactory/pipeline/Email Sender.json ================================================ { "name": "Email Sender", "properties": { "description": "Provide a simple abstract over the send email function with request body item exposed as pipeline parameters.", "activities": [ { "name": "Send Email", "description": "Use an Azure Function to perform an SMTP client email send operation.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "SendEmail", "method": "POST", "body": { "value": "{\n\"emailRecipients\": \"@{pipeline().parameters.Recipients}\",\n\"emailCcRecipients\": \"@{pipeline().parameters.CcRecipients}\",\n\"emailBccRecipients\": \"@{pipeline().parameters.BccRecipients}\",\n\"emailSubject\": \"@{pipeline().parameters.Subject}\",\n\"emailBody\": \"@{pipeline().parameters.Body}\",\n\"emailImportance\": \"@{pipeline().parameters.Importance}\"\n}", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } } ], "parameters": { "Recipients": { "type": "string" }, "CcRecipients": { "type": "string" }, "BccRecipients": { "type": "string" }, "Subject": { "type": "string" }, "Body": { "type": "string" }, "Importance": { "type": "string" } }, "folder": { "name": "_ProcFwk/_ProcFwkUtils" }, "annotations": [ "procfwk", "Utils" ] } } ================================================ FILE: DataFactory/pipeline/Intentional Error.json ================================================ { "name": "Intentional Error", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait1", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } }, { "name": "Raise Errors or Not", "description": "Framework development worker simulator.", "type": "IfCondition", "dependsOn": [ { "activity": "Wait1", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@equals(pipeline().parameters.RaiseErrors,'true')", "type": "Expression" }, "ifTrueActivities": [ { "name": "Call Fail Procedure", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[dbo].[FailProcedure]", "storedProcedureParameters": { "RaiseError": { "value": { "value": "@pipeline().parameters.RaiseErrors", "type": "Expression" }, "type": "String" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } } ], "parameters": { "RaiseErrors": { "type": "string", "defaultValue": "false" }, "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] } } ================================================ FILE: DataFactory/pipeline/Throw Exception.json ================================================ { "name": "Throw Exception", "properties": { "description": "Provide a simple way of throwing an exception within Data Factory using TSQL error handling.", "activities": [ { "name": "Raise Error Backup", "description": "Using a SQL database to raise an error/exception but wrapped up as a data factory pipeline. Error message information exposed as a pipeline parameter.", "type": "Lookup", "dependsOn": [ { "activity": "Raise Error", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderQuery": { "value": "RAISERROR('@{pipeline().parameters.Message}',16,1);", "type": "Expression" }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference" }, "firstRowOnly": false } }, { "name": "Raise Error", "description": "Using newer native activity raise an error/exception but wrapped up as a data factory pipeline. Error message information exposed as a pipeline parameter.", "type": "Fail", "dependsOn": [], "userProperties": [], "typeProperties": { "message": { "value": "@pipeline().parameters.Message", "type": "Expression" }, "errorCode": "16" } } ], "parameters": { "Message": { "type": "string" } }, "folder": { "name": "_ProcFwk/_ProcFwkUtils" }, "annotations": [ "procfwk", "Utils" ] } } ================================================ FILE: DataFactory/pipeline/Wait 1.json ================================================ { "name": "Wait 1", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait1", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] } } ================================================ FILE: DataFactory/pipeline/Wait 10.json ================================================ { "name": "Wait 10", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait10", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] } } ================================================ FILE: DataFactory/pipeline/Wait 2.json ================================================ { "name": "Wait 2", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait2", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] } } ================================================ FILE: DataFactory/pipeline/Wait 3.json ================================================ { "name": "Wait 3", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait3", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] } } ================================================ FILE: DataFactory/pipeline/Wait 4.json ================================================ { "name": "Wait 4", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait4", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] } } ================================================ FILE: DataFactory/pipeline/Wait 5.json ================================================ { "name": "Wait 5", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait5", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] } } ================================================ FILE: DataFactory/pipeline/Wait 6.json ================================================ { "name": "Wait 6", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait6", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] } } ================================================ FILE: DataFactory/pipeline/Wait 7.json ================================================ { "name": "Wait 7", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait7", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] } } ================================================ FILE: DataFactory/pipeline/Wait 8.json ================================================ { "name": "Wait 8", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait8", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] } } ================================================ FILE: DataFactory/pipeline/Wait 9.json ================================================ { "name": "Wait 9", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait9", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 15 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] } } ================================================ FILE: DataFactory/trigger/FunctionalTestingTrigger.json ================================================ { "name": "FunctionalTestingTrigger", "properties": { "description": "Used for functional testing of the framework in a dedicated environment.", "annotations": [ "procfwk" ], "runtimeState": "Stopped", "pipelines": [ { "pipelineReference": { "referenceName": "01-Grandparent", "type": "PipelineReference" } } ], "type": "ScheduleTrigger", "typeProperties": { "recurrence": { "frequency": "Hour", "interval": 2, "startTime": "2020-04-06T15:00:00Z", "timeZone": "UTC" } } } } ================================================ FILE: DeploymentTools/DataFactory/DeployProcFwkComponents.ps1 ================================================ function Publish-procfwkadf { Param( [Parameter(Mandatory)] [string]$resourceGroupName, [Parameter(Mandatory)] [string]$dataFactoryName, [Parameter(Mandatory)] [string]$region, [Parameter(Mandatory)] [string]$adfPath, [Parameter(Mandatory)] [string]$scriptPath ) #SPN for deploying ADF: $tenantId = [System.Environment]::GetEnvironmentVariable('AZURE_TENANT_ID') $subscriptionId = [System.Environment]::GetEnvironmentVariable('AZURE_SUBSCRIPTION_ID') $spId = [System.Environment]::GetEnvironmentVariable('AZURE_CLIENT_ID') $spKey = [System.Environment]::GetEnvironmentVariable('AZURE_CLIENT_SECRET') #Modules Import-Module -Name "Az" #Update-Module -Name "Az" Import-Module -Name "Az.DataFactory" #Update-Module -Name "Az.DataFactory" Import-Module -Name "azure.datafactory.tools" #Update-Module -Name "azure.datafactory.tools" Get-Module -Name "*DataFactory*" # Login as a Service Principal if ($spId) { $passwd = ConvertTo-SecureString $spKey -AsPlainText -Force $pscredential = New-Object System.Management.Automation.PSCredential($spId, $passwd) Connect-AzAccount -ServicePrincipal -Credential $pscredential -TenantId $tenantId | Out-Null } Get-AzContext # Get Deployment Objects and Params files $deploymentFilePath = Join-Path -Path $scriptPath -ChildPath "ProcFwkComponents.json" $configFilePath = Join-Path -Path $scriptPath -ChildPath "config-all.csv" $Env:SQLDatabase = "secretKeyToDbConnectionString" $opt = New-AdfPublishOption $deploymentObject = (Get-Content $deploymentFilePath) | ConvertFrom-Json $objectsToInclude = $deploymentObject.datasets + $deploymentObject.linkedServices + $deploymentObject.pipelines + $deploymentObject.triggers $objectsToInclude | ForEach-Object { $objName = $_.substring(1).Replace('.json', '').Replace('/', '.') $opt.Includes.Add($objName, "") } # Deployment of ADF $opt.CreateNewInstance = $true $opt.DeleteNotInSource = $false $opt.StopStartTriggers = $true Publish-AdfV2FromJson -RootFolder $adfPath ` -ResourceGroupName $resourceGroupName ` -DataFactoryName $dataFactoryName ` -Location $region ` -Option $opt ` -Stage $configFilePath } # Run function $VerbosePreference = 'Continue' $ErrorActionPreference = 'Stop' $scriptPath = Join-Path -Path (Get-Location) -ChildPath "\DeploymentTools\DataFactory" $AdfPath = Join-Path -Path (Get-Location) -ChildPath "DataFactory" Publish-procfwkadf -resourceGroupName 'rg-pademo' -dataFactoryName 'adf-metadata-driven-proc' -region 'uksouth' ` -adfPath "$AdfPath" -scriptPath "$scriptPath" ================================================ FILE: DeploymentTools/DataFactory/DeployProcFwkComponents_Old.ps1 ================================================ # Set global variables as required: $resourceGroupName = "ADF.procfwk" $dataFactoryName = "FrameworkFactoryDev" $region = "uksouth" #SPN for deploying ADF: $tenantId = [System.Environment]::GetEnvironmentVariable('AZURE_TENANT_ID') $subscriptionId = [System.Environment]::GetEnvironmentVariable('AZURE_SUBSCRIPTION_ID') $spId = [System.Environment]::GetEnvironmentVariable('AZURE_CLIENT_ID') $spKey = [System.Environment]::GetEnvironmentVariable('AZURE_CLIENT_SECRET') #Modules Import-Module -Name "Az" #Update-Module -Name "Az" Import-Module -Name "Az.DataFactory" #Update-Module -Name "Az.DataFactory" # Login as a Service Principal $passwd = ConvertTo-SecureString $spKey -AsPlainText -Force $pscredential = New-Object System.Management.Automation.PSCredential($spId, $passwd) Connect-AzAccount -ServicePrincipal -Credential $pscredential -TenantId $tenantId | Out-Null # Get Deployment Objects and Params files $scriptPath = "C:\Users\PaulAndrew\Source\GitHub\ADF.procfwk\DeploymentTools\DataFactory\" #$deploymentFilePath = $scriptPath + "\ProcFwkComponents.json" $deploymentFilePath = "C:\Users\PaulAndrew\Source\GitHub\ADF.procfwk\DeploymentTools\DataFactory\ProcFwkComponents.json" #Write-Host $scriptPath $deploymentObject = (Get-Content $deploymentFilePath) | ConvertFrom-Json <# Create Data Factory #> Write-Host "" Write-Host "-----------------------Data Factory-------------------------" $check = Get-AzDataFactoryV2 ` -ResourceGroupName $resourceGroupName ` -DataFactoryName $dataFactoryName ` -ErrorAction SilentlyContinue if($check -eq $null) { Write-Host "Creating Data Factory:" $dataFactoryName try { Set-AzDataFactoryV2 ` -ResourceGroupName $resourceGroupName ` -DataFactoryName $dataFactoryName ` -Location $region | Format-List | Out-Null Write-Host "...done" -ForegroundColor Green Write-Host "" } catch { Write-Host "Failed to created data factory:" $dataFactoryName -ForegroundColor Red Write-Host $_.Exception.Message Write-Host $_.Exception.ItemName Exit } } else { Write-Host "Data Factory $dataFactoryName already exists..." } <# Deploy Linked Services #> Write-Host "" Write-Host "----------------------Linked Services-----------------------" ForEach ($linkedService in $deploymentObject.linkedServices) { $componentPath = $scriptPath.Replace("DeploymentTools\","") + $linkedService $linkedServiceFile = (Get-Content $componentPath) | ConvertFrom-Json $linkedServiceName = $linkedServiceFile.name Write-Host "Deploying Linked Service:" $linkedServiceName try { Set-AzDataFactoryV2LinkedService ` -ResourceGroupName $resourceGroupName ` -DataFactoryName $dataFactoryName ` -Name $linkedServiceName ` -DefinitionFile $componentPath ` -Force | Format-List | Out-Null Write-Host "...done" -ForegroundColor Green Write-Host "" } catch { Write-Host "Failed to deploy linked service:" $linkedServiceName -ForegroundColor Green Write-Host $_.Exception.Message Write-Host $_.Exception.ItemName Exit } } <# Deploy Datasets #> Write-Host "" Write-Host "--------------------------Datasets--------------------------" ForEach ($dataSet in $deploymentObject.datasets) { $componentPath = $scriptPath.Replace("DeploymentTools\","") + $dataSet $datasetFile = (Get-Content $componentPath) | ConvertFrom-Json $datasetName = $datasetFile.name Write-Host "Deploying Dataset:" $datasetName try { Set-AzDataFactoryV2Dataset ` -ResourceGroupName $resourceGroupName ` -DataFactoryName $dataFactoryName ` -Name $datasetName ` -DefinitionFile $componentPath ` -Force | Format-List | Out-Null Write-Host "...done" -ForegroundColor Green Write-Host "" } catch { Write-Host "Failed to deploy dataset:" $datasetName -ForegroundColor Red Write-Host $_.Exception.Message Write-Host $_.Exception.ItemName Exit } } <# Deploy Pipelines #> Write-Host "" Write-Host "-------------------------Pipelines--------------------------" ForEach ($pipeline in $deploymentObject.pipelines) { $componentPath = $scriptPath.Replace("DeploymentTools\","") + $pipeline $pipelineFile = (Get-Content $componentPath) | ConvertFrom-Json $pipelineName = $pipelineFile.name Write-Host "Deploying Pipeline:" $pipelineName try { ##Bug in SDK means this native cmdlet can't be used if pipeline contains a Wait activity expression. <# Set-AzDataFactoryV2Pipeline ` -ResourceGroupName $resourceGroupName ` -DataFactoryName $dataFactoryName ` -Name $pipelineName ` -DefinitionFile $componentPath ` -Force | Format-List | Out-Null #> $body = (Get-Content -Path $componentPath | Out-String) $json = $body | ConvertFrom-Json New-AzResource ` -ResourceType 'Microsoft.DataFactory/factories/pipelines' ` -ResourceGroupName $resourceGroupName ` -Name "$dataFactoryName/$pipelineName" ` -ApiVersion "2018-06-01" ` -Properties $json ` -IsFullObject -Force | Out-Null Write-Host "...done" -ForegroundColor Green Write-Host "" } catch { Write-Host "Failed to deploy pipeline:" $pipelineName -ForegroundColor Red Write-Host $_.Exception.Message Write-Host $_.Exception.ItemName Exit } } <# Deploy Triggers #> Write-Host "" Write-Host "-------------------------Triggers---------------------------" ForEach ($trigger in $deploymentObject.triggers) { $componentPath = $scriptPath.Replace("DeploymentTools\","") + $trigger $triggerFile = (Get-Content $componentPath) | ConvertFrom-Json $triggerName = $triggerFile.name Write-Host "Deploying Trigger:" $triggerName $currentTrigger = Get-AzDataFactoryV2Trigger ` -ResourceGroupName $resourceGroupName ` -DataFactoryName $dataFactoryName ` -Name $triggerName ` -ErrorAction SilentlyContinue try { if($currentTrigger -ne $null) { #Stop trigger if already deployed as can't deploy over running trigger. Stop-AzDataFactoryV2Trigger ` -ResourceGroupName $resourceGroupName ` -DataFactoryName $dataFactoryName ` -Name $triggerName -Force | Out-Null } Set-AzDataFactoryV2Trigger ` -ResourceGroupName $resourceGroupName ` -DataFactoryName $dataFactoryName ` -Name $triggerName ` -DefinitionFile $componentPath ` -Force | Format-List | Out-Null Write-Host "...done" -ForegroundColor Green Write-Host "" } catch { Write-Host "Failed to deploy trigger:" $triggerName -ForegroundColor Red Write-Host $_.Exception.Message Write-Host $_.Exception.ItemName Exit } } ================================================ FILE: DeploymentTools/DataFactory/Get Pipelines.ps1 ================================================ # Set global variables as required: $resourceGroupName = "ADF.procfwk" $dataFactoryName = "FrameworkFactory" $region = "uksouth" #SPN for deploying ADF: $tenantId = [System.Environment]::GetEnvironmentVariable('AZURE_TENANT_ID') $subscriptionId = [System.Environment]::GetEnvironmentVariable('AZURE_SUBSCRIPTION_ID') $spId = [System.Environment]::GetEnvironmentVariable('AZURE_CLIENT_ID') $spKey = [System.Environment]::GetEnvironmentVariable('AZURE_CLIENT_SECRET') #Modules Import-Module -Name "Az" Import-Module -Name "Az.DataFactory" # Login as a Service Principal $passwd = ConvertTo-SecureString $spKey -AsPlainText -Force $pscredential = New-Object System.Management.Automation.PSCredential($spId, $passwd) Connect-AzAccount -ServicePrincipal -Credential $pscredential -TenantId $tenantId | Out-Null Get-AzDataFactoryV2Pipeline -DataFactoryName $dataFactoryName -ResourceGroupName $resourceGroupName ================================================ FILE: DeploymentTools/DataFactory/GlobalVars.ps1 ================================================ # Set global variables as required: $resourceGroupName = "ADF.procfwk" $dataFactoryName = "FrameworkFactory" $region = "uksouth" # ADF deployment from PS script .\DeploymentTools\DataFactory\DeployProcFwkComponents.ps1 ` -resourceGroupName "$resourceGroupName" ` -dataFactoryName "$dataFactoryName" ` -region "$region" .\DeploymentTools\DataFactory\PopulatePipelinesInDb.ps1 ` -SqlServerName '*****.database.windows.net' ` -SqlDatabaseName 'adfprocfwk' ` -SqlUser 'adm' ` -SqlPass '******' ` -resourceGroupName "$resourceGroupName" ` -dataFactoryName "$dataFactoryName" ` -region "$region" ================================================ FILE: DeploymentTools/DataFactory/PopulatePipelinesInDb.ps1 ================================================ Param( [Parameter(Mandatory)] [string]$SqlServerName, [Parameter(Mandatory)] [string]$SqlDatabaseName, [Parameter(Mandatory)] [string]$SqlUser, [Parameter(Mandatory)] [string]$SqlPass, [Parameter(Mandatory)] [string]$resourceGroupName, [Parameter(Mandatory)] [string]$dataFactoryName, [Parameter(Mandatory)] [string]$region ) $ErrorActionPreference = 'Stop' #Install-Module -Name SqlServer -AllowClobber Import-Module -Name SqlServer $pipelines = Get-AzDataFactoryV2Pipeline -DataFactoryName $dataFactoryName -ResourceGroupName $resourceGroupName foreach ($p in $pipelines) { Write-Host $p.name $query = "EXEC [procfwkHelpers].[AddPipelineViaPowerShell] '$resourceGroupName', '$dataFactoryName', '$($p.Name)';" Invoke-Sqlcmd -ServerInstance "$SqlServerName" -Database "$SqlDatabaseName" -Query "$query" -Username "$SqlUser" -Password "$SqlPass" } Write-Host "List of ADF pipelines has been populated into database." ================================================ FILE: DeploymentTools/DataFactory/ProcFwkComponents.json ================================================ { "linkedServices": [ "/linkedService/Keys.json", "/linkedService/FrameworkFunctions.json", "/linkedService/SupportDatabase.json" ], "datasets": [ "/dataset/GetSetMetadata.json" ], "pipelines": [ "/pipeline/Throw Exception.json", "/pipeline/Check For Running Pipeline.json", "/pipeline/Email Sender.json", "/pipeline/04-Infant.json", "/pipeline/03-Child.json", "/pipeline/02-Parent.json", "/pipeline/01-Grandparent.json" ], "triggers": [ "/trigger/FunctionalTestingTrigger.json" ] } ================================================ FILE: DeploymentTools/DataFactory/config-all.csv ================================================ type,name,path,value linkedService,SupportDatabase,typeProperties.connectionString.secretName,$($Env:SQLDatabase) ================================================ FILE: DeploymentTools/Deployment.targets ================================================ Debug AnyCPU bin\$(Configuration)\ false true false None obj\ $(BaseIntermediateOutputPath)\ $(BaseIntermediateOutputPath)$(Configuration)\ $(IntermediateOutputPath)ProjectReferences $(ProjectReferencesOutputPath)\ true false false Always Never false Build _GetDeploymentProjectContent; _CalculateContentOutputRelativePaths; _GetReferencedProjectsOutput; _CalculateArtifactStagingDirectory; _CopyOutputToArtifactStagingDirectory; Configuration=$(Configuration);Platform=$(Platform) $([System.IO.Path]::GetFileNameWithoutExtension('%(ProjectReference.Identity)')) $(OutDir) $(OutputPath) $(ArtifactStagingDirectory)\ $(ArtifactStagingDirectory)staging\ $(Build_StagingDirectory) <_OriginalIdentity>%(DeploymentProjectContentOutput.Identity) <_RelativePath>$(_OriginalIdentity.Replace('$(MSBuildProjectDirectory)', '')) $(_RelativePath) PrepareForRun ================================================ FILE: DeploymentTools/DeploymentTools.deployproj ================================================  Debug AnyCPU Release AnyCPU 3aead846-dbe4-45ad-97dd-37e94be009fd False ================================================ FILE: FactoryTesting/FactoryTesting.csproj ================================================ netcoreapp3.1 false all runtime; build; native; contentfiles; analyzers; buildtransitive ================================================ FILE: FactoryTesting/Helpers/CoverageHelper.cs ================================================ using FactoryTesting.Helpers; using Microsoft.Azure.Management.DataFactory.Models; using NUnit.Framework; using System; using System.Collections.Generic; using System.Threading.Tasks; namespace FactoryTesting.Helpers { public class CoverageHelper : DatabaseHelper where T : CoverageHelper { public new async Task RunPipeline(string pipelineName) { var callStack = new System.Diagnostics.StackTrace(); await base.RunPipeline(pipelineName); if (ReportTestCoverage) foreach (var ar in await GetActivityRuns()) RecordActivityRun(ar, callStack.ToString()); } public new async Task RunPipelineAndCancel(string pipelineName) { var callStack = new System.Diagnostics.StackTrace(); await base.RunPipelineAndCancel(pipelineName); } public bool ReportTestCoverage { get { try { var measure = GetSetting("ReportTestCoverage"); if (measure == "true") return true; } catch (Exception) { } return false; } } private void RecordActivityRun(ActivityRun ar, string context) { var parameters = new Dictionary { ["@pipelineName"] = ar.PipelineName, ["@activityName"] = ar.ActivityName, ["@context"] = context }; ExecuteStoredProcedure("test.RecordActivityRun", parameters); } } } [SetUpFixture] public class CoverageHelperSetup : CoverageHelper { [OneTimeSetUp] public async Task SetupCoverageHelper() { if (ReportTestCoverage) { WithEmptyTable("test.ActivityRun"); WithEmptyTable("test.PipelineActivity"); foreach (var p in await GetPipelines()) if (p.Activities != null) foreach (var a in p.Activities) RecordActivity(p.Name, a); } TearDown(); } private void RecordActivity(string pipelineName, Activity act) { var parameters = new Dictionary { ["@pipelineName"] = pipelineName, ["@activityName"] = act.Name }; ExecuteStoredProcedure("test.RecordActivity", parameters); if (act is ForEachActivity) foreach (var a in ((ForEachActivity)act).Activities) RecordActivity(pipelineName, a); if (act is UntilActivity) foreach (var a in ((UntilActivity)act).Activities) RecordActivity(pipelineName, a); if (act is IfConditionActivity) { foreach (var a in ((IfConditionActivity)act).IfTrueActivities) RecordActivity(pipelineName, a); foreach (var a in ((IfConditionActivity)act).IfFalseActivities) RecordActivity(pipelineName, a); } if (act is SwitchActivity) foreach (var c in ((SwitchActivity)act).Cases) foreach (var a in c.Activities) RecordActivity(pipelineName, a); } } ================================================ FILE: FactoryTesting/Helpers/DataFactoryHelper.cs ================================================ using Microsoft.Azure.Management.DataFactory; using Microsoft.Azure.Management.DataFactory.Models; using Microsoft.IdentityModel.Clients.ActiveDirectory; using Microsoft.Rest; using System; using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; namespace FactoryTesting.Helpers { public class DataFactoryHelper : SettingsHelper where T : DataFactoryHelper { private readonly string _adfName; private readonly string _rgName; private DataFactoryManagementClient _adfClient; private async Task InitialiseClient() { if (_adfClient != null) return; var context = new AuthenticationContext("https://login.windows.net/" + GetSetting("AZURE_TENANT_ID")); var cc = new ClientCredential(GetSetting("AZURE_CLIENT_ID"), GetSetting("AZURE_CLIENT_SECRET")); var authResult = await context.AcquireTokenAsync("https://management.azure.com/", cc); var cred = new TokenCredentials(authResult.AccessToken); _adfClient = new DataFactoryManagementClient(cred) { SubscriptionId = GetSetting("AZURE_SUBSCRIPTION_ID") }; } public async Task TriggerPipeline(string pipelineName, IDictionary parameters) { await InitialiseClient(); var response = await _adfClient.Pipelines.CreateRunWithHttpMessagesAsync(_rgName, _adfName, pipelineName, parameters: parameters); return response.Body.RunId; } public async Task> GetPipelines() { await InitialiseClient(); var page = await _adfClient.Pipelines.ListByFactoryAsync(_rgName, _adfName); var pipelines = page.ToList(); while (!string.IsNullOrWhiteSpace(page.NextPageLink)) { page = await _adfClient.Pipelines.ListByFactoryNextAsync(page.NextPageLink); pipelines.AddRange(page.ToList()); } return pipelines; } public async Task> GetActivityRuns(string pipelineRunId) { await InitialiseClient(); var filter = new RunFilterParameters(DateTime.MinValue, DateTime.UtcNow); var arqr = await _adfClient.ActivityRuns.QueryByPipelineRunAsync(_rgName, _adfName, pipelineRunId, filter); var activityRuns = arqr.Value.ToList(); while (!string.IsNullOrWhiteSpace(arqr.ContinuationToken)) { filter.ContinuationToken = arqr.ContinuationToken; arqr = await _adfClient.ActivityRuns.QueryByPipelineRunAsync(_rgName, _adfName, pipelineRunId, filter); activityRuns.AddRange(arqr.Value); } return activityRuns; } public virtual void TearDown() { _adfClient?.Dispose(); } public async Task GetRunStatus(string pipelineRunId) { await InitialiseClient(); var run = await _adfClient.PipelineRuns.GetAsync(_rgName, _adfName, pipelineRunId); return run.Status; } public async Task GetRunStatus(string pipelineRunId, string adfName) { await InitialiseClient(); var run = await _adfClient.PipelineRuns.GetAsync(_rgName, adfName, pipelineRunId); return run.Status; } public async Task IsInProgress(string pipelineRunId) { await InitialiseClient(); var status = await GetRunStatus(pipelineRunId); return status == "Queued" || status == "InProgress" || status == "Cancelling"; } public async Task IsInProgress(string pipelineRunId, string adfName) { await InitialiseClient(); var status = await GetRunStatus(pipelineRunId, adfName); return status == "Queued" || status == "InProgress" || status == "Cancelling"; } public async Task IsQueued(string pipelineRunId) { await InitialiseClient(); var status = await GetRunStatus(pipelineRunId); return status == "Queued"; } public async Task IsCancelling(string pipelineRunId) { await InitialiseClient(); var status = await GetRunStatus(pipelineRunId); return status == "Cancelling" || status == "Canceling"; //microsoft typo } public async Task IsCancelling(string pipelineRunId, string adfName) { await InitialiseClient(); var status = await GetRunStatus(pipelineRunId, adfName); return status == "Cancelling" || status == "Canceling"; //microsoft typo } public async Task CancelRunningPipeline(string pipelineRunId, bool recurseCancel = true) { await InitialiseClient(); if (await IsInProgress(pipelineRunId)) { _adfClient.PipelineRuns.Cancel(_rgName, _adfName, pipelineRunId, recurseCancel); while (await IsCancelling(pipelineRunId)) Thread.Sleep(2000); } else { throw new Exception("Pipeline is not in a state that can be cancelled."); } string status = await GetRunStatus(pipelineRunId); return status; } public async Task CancelRunningPipeline(string pipelineRunId, string adfName, bool recurseCancel = true) { await InitialiseClient(); if (await IsInProgress(pipelineRunId, adfName)) { _adfClient.PipelineRuns.Cancel(_rgName, adfName, pipelineRunId, recurseCancel); while (await IsCancelling(pipelineRunId, adfName)) Thread.Sleep(2000); } else { throw new Exception("Pipeline is not in a state that can be cancelled."); } string status = await GetRunStatus(pipelineRunId, adfName); return status; } public DataFactoryHelper() { _adfName = GetSetting("DataFactoryName"); _rgName = GetSetting("DataFactoryResourceGroup"); } } } ================================================ FILE: FactoryTesting/Helpers/DatabaseHelper.cs ================================================ using System.Collections.Generic; using System.Data; using System.Data.SqlClient; namespace FactoryTesting.Helpers { public class DatabaseHelper : PipelineRunHelper where T : DatabaseHelper { public SqlConnection _conn; public DatabaseHelper() { _conn = new SqlConnection(GetSetting(GetSetting("MetadataDbConnectionStringSecretName"))); _conn.Open(); } public T WithEmptyTable(string tableName) { using (var cmd = new SqlCommand($"TRUNCATE TABLE {tableName}", _conn)) cmd.ExecuteNonQuery(); return (T)this; } public int RowCount(string tableName, string where = "", string equals = "") { using (var cmd = new SqlCommand($"SELECT COUNT(*) FROM {tableName}" + (where?.Length == 0 ? "" : $" WHERE {where} = '{equals.Replace("'", "''")}'") , _conn)) using (var reader = cmd.ExecuteReader()) { reader.Read(); return reader.GetInt32(0); } } public string ColumnData(string tableName, string columnName, char separator = ',') { using (var cmd = new SqlCommand($"SELECT STRING_AGG([{columnName}],'{separator}') FROM {tableName}", _conn)) using (var reader = cmd.ExecuteReader()) { reader.Read(); return reader.GetString(0); } } public void ExecuteStoredProcedure(string spName, Dictionary parameters = null) { using (var cmd = new SqlCommand(spName, _conn)) { cmd.CommandType = CommandType.StoredProcedure; if (parameters != null) foreach (string parameterName in parameters.Keys) cmd.Parameters.Add(new SqlParameter(parameterName, parameters[parameterName])); cmd.ExecuteNonQuery(); } } public void ExecuteNonQuery(string sql) { using (var cmd = new SqlCommand(sql, _conn)) cmd.ExecuteNonQuery(); } public void AddTenantAndSubscription(string tenantId = null, string subscriptionId = null) { if (string.IsNullOrEmpty(tenantId)) tenantId = GetSetting("AZURE_TENANT_ID"); if (string.IsNullOrEmpty(subscriptionId)) subscriptionId = GetSetting("AZURE_SUBSCRIPTION_ID"); ExecuteNonQuery($"INSERT INTO[procfwk].[Tenants] ([TenantId],[Name],[Description]) VALUES ('{tenantId}', 'mrpaulandrew.com', NULL);"); ExecuteNonQuery($"INSERT INTO [procfwk].[Subscriptions] ([SubscriptionId],[Name],[Description],[TenantId]) VALUES ('{subscriptionId}', 'Microsoft Sponsored Subscription', NULL, '{tenantId}');"); ExecuteNonQuery($"UPDATE [procfwk].[Orchestrators] SET [SubscriptionId] = '{subscriptionId}';"); } public void AddWorkerSPNStoredInDatabase(string workerFactoryName, string orchestratorType = "ADF") { ExecuteNonQuery("UPDATE [procfwk].[Properties] SET [PropertyValue] = 'StoreInDatabase' WHERE [PropertyName] = 'SPNHandlingMethod';"); var parameters = new Dictionary { ["@OrchestratorName"] = workerFactoryName, ["@OrchestratorType"] = orchestratorType, ["@PrincipalIdValue"] = GetSetting("AZURE_CLIENT_ID"), ["@PrincipalSecretValue"] = GetSetting("AZURE_CLIENT_SECRET"), ["@PrincipalName"] = GetSetting("AZURE_CLIENT_NAME") }; ExecuteStoredProcedure("[procfwkHelpers].[AddServicePrincipalWrapper]", parameters); } public void AddWorkerSPNStoredInKeyVault(string workerFactoryName, string orchestratorType = "ADF") { ExecuteNonQuery("UPDATE [procfwk].[Properties] SET [PropertyValue] = 'StoreInKeyVault' WHERE [PropertyName] = 'SPNHandlingMethod';"); var parameters = new Dictionary { ["@OrchestratorName"] = workerFactoryName, ["@OrchestratorType"] = orchestratorType, ["@PrincipalIdValue"] = GetSetting("AZURE_CLIENT_ID_URL"), ["@PrincipalSecretValue"] = GetSetting("AZURE_CLIENT_SECRET_URL"), ["@PrincipalName"] = GetSetting("AZURE_CLIENT_NAME") }; ExecuteStoredProcedure("[procfwkHelpers].[AddServicePrincipalWrapper]", parameters); } public void AddBasicMetadata() { ExecuteStoredProcedure("[procfwkTesting].[ResetMetadata]", null); ExecuteNonQuery("UPDATE [procfwk].[Orchestrators] SET [IsFrameworkOrchestrator] = '0';"); ExecuteNonQuery($"UPDATE [procfwk].[Orchestrators] SET [IsFrameworkOrchestrator] = '1' WHERE [OrchestratorName] = '{GetSetting("DataFactoryName")}';"); } public override void TearDown() { ExecuteStoredProcedure("[procfwkTesting].[CleanUpMetadata]", null); _conn?.Dispose(); base.TearDown(); } } } ================================================ FILE: FactoryTesting/Helpers/PipelineRunHelper.cs ================================================ using Microsoft.Azure.Management.DataFactory.Models; using Newtonsoft.Json.Linq; using System; using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; namespace FactoryTesting.Helpers { public class PipelineRunHelper : DataFactoryHelper where T : PipelineRunHelper { private readonly Dictionary _parameters; private List _activityRuns; private bool _hasRun; public string RunId { get; private set; } public string RunOutcome { get; private set; } public PipelineRunHelper() { RunId = "Unknown"; RunOutcome = "Unknown"; _hasRun = false; _parameters = new Dictionary(); } public T WithParameter(string name, object value) { _parameters[name] = value; return (T)this; } public async Task RunPipeline(string pipelineName) { if (_hasRun) throw new Exception("RunPipeline() can only be called once per instance lifetime"); _hasRun = true; RunId = await TriggerPipeline(pipelineName, _parameters); while (await IsInProgress(RunId)) Thread.Sleep(2000); RunOutcome = await GetRunStatus(RunId); } public async Task RunPipelineAndCancel(string pipelineName) { if (_hasRun) throw new Exception("RunPipeline() can only be called once per instance lifetime"); _hasRun = true; RunId = await TriggerPipeline(pipelineName, _parameters); while (await IsQueued(RunId)) Thread.Sleep(2000); await CancelRunningPipeline(RunId, true); RunOutcome = await GetRunStatus(RunId); } public async Task> GetActivityRuns() { await InitialiseActivityRuns(); return _activityRuns; } public async Task GetActivityRunCount(string pattern = ".*") { await InitialiseActivityRuns(); Regex rgx = new Regex(pattern); return _activityRuns.Where(ar => rgx.IsMatch(ar.ActivityName)).Count(); } private async Task InitialiseActivityRuns() { if (_activityRuns == null) _activityRuns = await GetActivityRuns(RunId); } public async Task GetActivityOutput(string activityName, string propertyPath = "$") { await InitialiseActivityRuns(); string output = _activityRuns.Where(ar => ar.ActivityName == activityName).FirstOrDefault().Output.ToString(); var obj = JObject.Parse(output); return obj.SelectToken(propertyPath).ToString(); } } } ================================================ FILE: FactoryTesting/Helpers/SettingsHelper.cs ================================================ using Azure.Identity; using Azure.Security.KeyVault.Secrets; using NUnit.Framework; using System; namespace FactoryTesting.Helpers { public class SettingsHelper { public string GetSetting(string settingName) { // return environment variable "settingName", if present var value = Environment.GetEnvironmentVariable(settingName); if (value?.Length > 0) return value; // return value of runsettings parameter "settingName", if present value = TestContext.Parameters[settingName]; if (value?.Length > 0) return value; // if a key vault is specified, return the value of secret "settingName", if present if (_keyVaultClient != null) { value = _keyVaultClient.GetSecret(settingName).Value.Value; if (value?.Length > 0) return value; } throw new Exception($"Test setting '{settingName}' not found"); } private readonly SecretClient _keyVaultClient; public SettingsHelper() { var kvUrl = TestContext.Parameters["KeyVaultUrl"]; if (kvUrl?.Length > 0) _keyVaultClient = new SecretClient(new Uri(kvUrl), new DefaultAzureCredential()); } } } ================================================ FILE: FactoryTesting/Pipelines/01-Grandparent/Given300WorkerPipelines.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Grandparent { public class Given300WorkerPipelines { private GrandparentHelper _helper; [OneTimeSetUp] public async Task WhenPipelineRun() { _helper = new GrandparentHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .With300WorkerPipelinesEnabled() .WithSPNInKeyVault("WorkersFactory"); await _helper.RunPipeline(); } #region Integration tests [Test] public void ThenPipelineOutcomeIsSucceeded() { _helper.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenCurrentExecutionTableIsEmpty() { _helper.RowCount("procfwk.CurrentExecution").Should().Be(0); } [Test] public void Then300ExecutionLogRecords() { _helper.RowCount("procfwk.ExecutionLog").Should().Be(300); } #endregion [OneTimeTearDown] public void TearDown() { _helper?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/01-Grandparent/Given600WorkerPipelineConcurrentBatches.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Grandparent { public class Given600WorkerPipelineConcurrentBatches { private GrandparentHelper _helperFirstBatch; private GrandparentHelper _helperSecondBatch; [OneTimeSetUp] public async Task WhenPipelineRun() { _helperFirstBatch = new GrandparentHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .With300WorkerPipelinesEnabled() .WithBatchExecutionHandling() .With300WorkerPipelineBatches() .WithSPNInKeyVault("WorkersFactory") .WithParameter("BatchName", "0to300"); _helperSecondBatch = new GrandparentHelper() .WithParameter("BatchName", "301to600"); var firstBatch = _helperFirstBatch.RunPipeline(); var secondBatch = _helperSecondBatch.RunPipeline(); await Task.WhenAll(firstBatch, secondBatch); } #region Integration tests [Test] public void ThenFirstBatchPipelineOutcomeIsSucceeded() { _helperFirstBatch.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenSecondBatchPipelineOutcomeIsSucceeded() { _helperSecondBatch.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenTwoBatchExecutionSuccessRecords() { _helperFirstBatch.RowCount("procfwk.BatchExecution", where: "BatchStatus", equals: "Success").Should().Be(2); } [Test] public void ThenCurrentExecutionTableIsEmpty() { _helperFirstBatch.RowCount("procfwk.CurrentExecution").Should().Be(0); } [Test] public void Then300ExecutionLogRecords() { _helperFirstBatch.RowCount("procfwk.ExecutionLog").Should().Be(600); } #endregion [OneTimeTearDown] public void TearDown() { _helperFirstBatch?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/01-Grandparent/GivenOneWorkerPipeline.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Grandparent { public class GivenOneWorkerPipeline { private GrandparentHelper _helper; [OneTimeSetUp] public async Task WhenPipelineRun() { _helper = new GrandparentHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .WithSPNInDatabase("FrameworkFactory") .WithEmptyExecutionTables() .WithSimpleFailureHandling() .WithOneWorkerPipelineEnabled(); await _helper.RunPipeline(); } #region Integration tests [Test] public void ThenPipelineOutcomeIsSucceeded() { _helper.RunOutcome.Should().Be("Succeeded"); } #endregion [OneTimeTearDown] public void TearDown() { _helper?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/01-Grandparent/GrandparentHelper.cs ================================================ using FactoryTesting.Helpers; using System.Collections.Generic; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Grandparent { class GrandparentHelper : CoverageHelper { public async Task RunPipeline() { await RunPipeline("01-Grandparent"); } public GrandparentHelper WithTenantAndSubscriptionIds() { AddTenantAndSubscription(); return this; } public GrandparentHelper WithSPNInDatabase(string workerFactoryName) { AddWorkerSPNStoredInDatabase(workerFactoryName); return this; } public GrandparentHelper WithSPNInKeyVault(string workerFactoryName) { AddWorkerSPNStoredInKeyVault(workerFactoryName); return this; } public GrandparentHelper WithBasicMetadata() { AddBasicMetadata(); return this; } public GrandparentHelper WithEmptyExecutionTables() { WithEmptyTable("procfwk.CurrentExecution"); WithEmptyTable("procfwk.ExecutionLog"); WithEmptyTable("procfwk.ErrorLog"); return this; } public GrandparentHelper WithBatchExecutionHandling() { ExecuteNonQuery(@$"UPDATE [procfwk].[Properties] SET [PropertyValue] = '1' WHERE [PropertyName] = 'UseExecutionBatches'"); return this; } public GrandparentHelper WithSimpleFailureHandling() { ExecuteNonQuery("UPDATE [procfwk].[Properties] SET [PropertyValue] = 'Simple' WHERE [PropertyName] = 'FailureHandling'"); return this; } public GrandparentHelper WithOneWorkerPipelineEnabled() { ExecuteNonQuery("UPDATE [procfwk].[Pipelines] SET [Enabled] = 0 WHERE [PipelineId] > 1"); return this; } public GrandparentHelper With300WorkerPipelinesEnabled() { ExecuteStoredProcedure("[procfwkTesting].[Add300WorkerPipelines]", null); return this; } public GrandparentHelper With20BatchesFor1000WorkersEnabled() { ExecuteStoredProcedure("[procfwkTesting].[Add20BatchesFor1000Workers]", null); return this; } public GrandparentHelper WithCustom() { ExecuteStoredProcedure("[dbo].[PaulTemp]", null); return this; } public GrandparentHelper With300WorkerPipelineBatches() { ExecuteStoredProcedure("[procfwkTesting].[Add300WorkerPipelineBatches]", null); return this; } } } ================================================ FILE: FactoryTesting/Pipelines/02-Parent/Given20ConcurrentBatchesFor1000WorkerPipelines.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Parent { public class Given20ConcurrentBatchesFor1000WorkerPipelines { private ParentHelper _helperBatchOne; private ParentHelper _helperBatchTwo; private ParentHelper _helperBatchEleven; private ParentHelper _helperBatchTwelve; private ParentHelper _helperBatchTwenty; private ParentHelper _helperBatchFifteen; private ParentHelper _helperBatchSeven; private ParentHelper _helperBatchNine; private ParentHelper _helperBatchEight; private ParentHelper _helperBatchSixteen; private ParentHelper _helperBatchFive; private ParentHelper _helperBatchSix; private ParentHelper _helperBatchThirteen; private ParentHelper _helperBatchNineteen; private ParentHelper _helperBatchFour; private ParentHelper _helperBatchEighteen; private ParentHelper _helperBatchThree; private ParentHelper _helperBatchFourteen; private ParentHelper _helperBatchTen; private ParentHelper _helperBatchSeventeen; [OneTimeSetUp] public async Task WhenPipelineRun() { /* _helperBatchOne = new ParentHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .WithBatchExecutionHandling() .With20BatchesFor1000WorkersEnabled() .WithSPNInKeyVault("WorkersFactory") .WithParameter("BatchName", "One"); */ _helperBatchOne = new ParentHelper() .WithCustom() .WithParameter("BatchName", "One"); _helperBatchTwo = new ParentHelper().WithParameter("BatchName", "Two"); _helperBatchEleven = new ParentHelper().WithParameter("BatchName", "Eleven"); _helperBatchTwelve = new ParentHelper().WithParameter("BatchName", "Twelve"); _helperBatchTwenty = new ParentHelper().WithParameter("BatchName", "Twenty"); _helperBatchFifteen = new ParentHelper().WithParameter("BatchName", "Fifteen"); _helperBatchSeven = new ParentHelper().WithParameter("BatchName", "Seven"); _helperBatchNine = new ParentHelper().WithParameter("BatchName", "Nine"); _helperBatchEight = new ParentHelper().WithParameter("BatchName", "Eight"); _helperBatchSixteen = new ParentHelper().WithParameter("BatchName", "Sixteen"); _helperBatchFive = new ParentHelper().WithParameter("BatchName", "Five"); _helperBatchSix = new ParentHelper().WithParameter("BatchName", "Six"); _helperBatchThirteen = new ParentHelper().WithParameter("BatchName", "Thirteen"); _helperBatchNineteen = new ParentHelper().WithParameter("BatchName", "Nineteen"); _helperBatchFour = new ParentHelper().WithParameter("BatchName", "Four"); _helperBatchEighteen = new ParentHelper().WithParameter("BatchName", "Eighteen"); _helperBatchThree = new ParentHelper().WithParameter("BatchName", "Three"); _helperBatchFourteen = new ParentHelper().WithParameter("BatchName", "Fourteen"); _helperBatchTen = new ParentHelper().WithParameter("BatchName", "Ten"); _helperBatchSeventeen = new ParentHelper().WithParameter("BatchName", "Seventeen"); var batchOne = _helperBatchOne.RunPipeline(); var batchTwo = _helperBatchTwo.RunPipeline(); var batchEleven = _helperBatchEleven.RunPipeline(); var batchTwelve = _helperBatchTwelve.RunPipeline(); var batchTwenty = _helperBatchTwenty.RunPipeline(); var batchFifteen = _helperBatchFifteen.RunPipeline(); var batchSeven = _helperBatchSeven.RunPipeline(); var batchNine = _helperBatchNine.RunPipeline(); var batchEight = _helperBatchEight.RunPipeline(); var batchSixteen = _helperBatchSixteen.RunPipeline(); var batchFive = _helperBatchFive.RunPipeline(); var batchSix = _helperBatchSix.RunPipeline(); var batchThirteen = _helperBatchThirteen.RunPipeline(); var batchNineteen = _helperBatchNineteen.RunPipeline(); var batchFour = _helperBatchFour.RunPipeline(); var batchEighteen = _helperBatchEighteen.RunPipeline(); var batchThree = _helperBatchThree.RunPipeline(); var batchFourteen = _helperBatchFourteen.RunPipeline(); var batchTen = _helperBatchTen.RunPipeline(); var batchSeventeen = _helperBatchSeventeen.RunPipeline(); //await _helperBatchOne.RunPipeline(); await Task.WhenAll( batchOne, batchTwo, batchEleven, batchTwelve, batchTwenty, batchFifteen, batchSeven, batchNine, batchEight, batchSixteen, batchFive, batchSix, batchThirteen, batchNineteen, batchFour, batchEighteen, batchThree, batchFourteen, batchTen, batchSeventeen ); } #region Integration tests [Test] public void ThenOneBatchPipelineOutcomeIsSucceeded() { _helperBatchOne.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenTwoBatchPipelineOutcomeIsSucceeded() { _helperBatchTwo.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenElevenBatchPipelineOutcomeIsSucceeded() { _helperBatchEleven.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenTwelveBatchPipelineOutcomeIsSucceeded() { _helperBatchTwelve.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenTwentyBatchPipelineOutcomeIsSucceeded() { _helperBatchTwenty.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenFifteenBatchPipelineOutcomeIsSucceeded() { _helperBatchFifteen.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenSevenBatchPipelineOutcomeIsSucceeded() { _helperBatchSeven.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenNineBatchPipelineOutcomeIsSucceeded() { _helperBatchNine.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenEightBatchPipelineOutcomeIsSucceeded() { _helperBatchEight.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenSixteenBatchPipelineOutcomeIsSucceeded() { _helperBatchSixteen.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenFiveBatchPipelineOutcomeIsSucceeded() { _helperBatchFive.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenSixBatchPipelineOutcomeIsSucceeded() { _helperBatchSix.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenThirteenBatchPipelineOutcomeIsSucceeded() { _helperBatchThirteen.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenNineteenBatchPipelineOutcomeIsSucceeded() { _helperBatchNineteen.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenFourBatchPipelineOutcomeIsSucceeded() { _helperBatchFour.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenEighteenBatchPipelineOutcomeIsSucceeded() { _helperBatchEighteen.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenThreeBatchPipelineOutcomeIsSucceeded() { _helperBatchThree.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenFourteenBatchPipelineOutcomeIsSucceeded() { _helperBatchFourteen.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenTenBatchPipelineOutcomeIsSucceeded() { _helperBatchTen.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenSeventeenBatchPipelineOutcomeIsSucceeded() { _helperBatchSeventeen.RunOutcome.Should().Be("Succeeded"); } [Test] public void Then1000ExecutionLogRecords() { _helperBatchOne.RowCount("procfwk.ExecutionLog").Should().Be(1000); } #endregion /* [OneTimeTearDown] public void TearDown() { _helperBatchOne?.TearDown(); } */ } } ================================================ FILE: FactoryTesting/Pipelines/02-Parent/GivenAlreadyRunning.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Parent { public class GivenAlreadyRunning { private ParentHelper _helperRunOne; private ParentHelper _helperRunTwo; [OneTimeSetUp] public async Task WhenPipelineRun() { _helperRunOne = new ParentHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .WithSPNInDatabase("FrameworkFactory") .WithEmptyExecutionTables() .WithoutSimulatedError() .WithFailureHandling("Simple"); _helperRunTwo = new ParentHelper(); var firstRun = _helperRunOne.RunPipeline(); var secondRun = _helperRunTwo.RunPipeline(15000); //15 second delay in run await Task.WhenAll(firstRun, secondRun); } [Test] public void ThenFirstPipelineOutcomeIsSucceeded() { _helperRunOne.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenSecondPipelineOutcomeIsFailed() { _helperRunTwo.RunOutcome.Should().Be("Failed"); } [Test] public void ThenCurrentExecutionTableIsEmpty() { _helperRunOne.RowCount("procfwk.CurrentExecution").Should().Be(0); } [Test] public void ThenElevenExecutionLogRecords() { _helperRunOne.RowCount("procfwk.ExecutionLog").Should().Be(11); } [OneTimeTearDown] public void TearDown() { _helperRunOne?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/02-Parent/GivenBatchExecutionsForConcurrentBatches.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Parent { public class GivenBatchExecutionsForConcurrentBatches { private ParentHelper _helperFirstBatch; private ParentHelper _helperSecondBatch; [OneTimeSetUp] public async Task WhenPipelineRun() { _helperFirstBatch = new ParentHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .WithSPNInDatabase("FrameworkFactory") .WithEmptyExecutionTables() .WithBatchExecutionHandling() .WithStagesEnabled() .WithPipelinesEnabled() .WithParameter("BatchName", "Hourly"); _helperSecondBatch = new ParentHelper() .WithParameter("BatchName", "Daily"); var firstBatch = _helperFirstBatch.RunPipeline(); var secondBatch = _helperSecondBatch.RunPipeline(); await Task.WhenAll(firstBatch, secondBatch); } #region Integration tests [Test] public void ThenFirstBatchPipelineOutcomeIsSucceeded() { _helperFirstBatch.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenSecondBatchPipelineOutcomeIsSucceeded() { _helperSecondBatch.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenCurrentExecutionTableIsEmpty() { _helperFirstBatch.RowCount("procfwk.CurrentExecution").Should().Be(0); } [Test] public void ThenFifteenExecutionLogSuccessRecords() { _helperFirstBatch.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Success").Should().Be(15); } [Test] public void ThenTwoBatchExecutionSuccessRecords() { _helperFirstBatch.RowCount("procfwk.BatchExecution", where: "BatchStatus", equals: "Success").Should().Be(2); } #endregion [OneTimeTearDown] public void TearDown() { _helperFirstBatch?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/02-Parent/GivenBatchExecutionsForConcurrentBatchesAlreadyRunning.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Parent { public class GivenBatchExecutionsForConcurrentBatchesAlreadyRunning { private ParentHelper _helperFirstBatch; private ParentHelper _helperSecondBatch; [OneTimeSetUp] public async Task WhenPipelineRun() { _helperFirstBatch = new ParentHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .WithSPNInDatabase("FrameworkFactory") .WithEmptyExecutionTables() .WithBatchExecutionHandling() .WithStagesEnabled() .WithPipelinesEnabled() .WithParameter("BatchName", "Hourly"); _helperSecondBatch = new ParentHelper() .WithParameter("BatchName", "Hourly"); var firstBatch = _helperFirstBatch.RunPipeline(); var secondBatch = _helperSecondBatch.RunPipeline(15000); //15 second delay in run await Task.WhenAll(firstBatch, secondBatch); } #region Integration tests [Test] public void ThenFirstBatchPipelineOutcomeIsSucceeded() { _helperFirstBatch.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenSecondBatchPipelineOutcomeIsFailed() { _helperSecondBatch.RunOutcome.Should().Be("Failed"); } [Test] public void ThenCurrentExecutionTableIsEmpty() { _helperFirstBatch.RowCount("procfwk.CurrentExecution").Should().Be(0); } [Test] public void ThenFourExecutionLogSuccessRecords() { _helperFirstBatch.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Success").Should().Be(4); } [Test] public void ThenOneBatchExecutionSuccessRecord() { _helperFirstBatch.RowCount("procfwk.BatchExecution", where: "BatchStatus", equals: "Success").Should().Be(1); } #endregion [OneTimeTearDown] public void TearDown() { _helperFirstBatch?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/02-Parent/GivenBatchExecutionsForConcurrentBatchesWithSimpleFailureHandling.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Parent { public class GivenBatchExecutionsForConcurrentBatchesWithSimpleFailureHandling { private ParentHelper _helperFirstBatch; private ParentHelper _helperSecondBatch; [OneTimeSetUp] public async Task WhenPipelineRun() { _helperFirstBatch = new ParentHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .WithSPNInDatabase("FrameworkFactory") .WithEmptyExecutionTables() .WithBatchExecutionHandling() .WithStagesEnabled() .WithPipelinesEnabled() .WithParameter("BatchName", "Hourly"); _helperSecondBatch = new ParentHelper() .WithSimulatedError() .WithFailureHandling("Simple") .WithParameter("BatchName", "Daily"); var firstBatch = _helperFirstBatch.RunPipeline(); var secondBatch = _helperSecondBatch.RunPipeline(); await Task.WhenAll(firstBatch, secondBatch); } #region Integration tests //batch one - hourly: [Test] public void ThenFirstBatchPipelineOutcomeIsSucceeded() { _helperFirstBatch.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenOneBatchExecutionSuccessRecord() { _helperFirstBatch.RowCount("procfwk.BatchExecution", where: "BatchStatus", equals: "Success").Should().Be(1); } [Test] public void ThenFourExecutionLogSuccessRecords() { _helperFirstBatch.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Success").Should().Be(4); } //batch two - daily: [Test] public void ThenSecondBatchPipelineOutcomeIsFailed() { _helperSecondBatch.RunOutcome.Should().Be("Failed"); } [Test] public void ThenOneBatchExecutionStoppedRecord() { _helperSecondBatch.RowCount("procfwk.BatchExecution", where: "BatchStatus", equals: "Stopped").Should().Be(1); } [Test] public void ThenThreeExecutionsSucceeded() { _helperSecondBatch.RowCount("procfwk.CurrentExecution", where: "PipelineStatus", equals: "Success").Should().Be(3); } [Test] public void ThenOneExecutionFailed() { _helperSecondBatch.RowCount("procfwk.CurrentExecution", where: "PipelineStatus", equals: "Failed").Should().Be(1); } [Test] public void ThenSevenExecutionsBlocked() { _helperSecondBatch.RowCount("procfwk.CurrentExecution", where: "PipelineStatus", equals: "Blocked").Should().Be(7); } [Test] public void ThenOneExecutionLogRecord() { _helperSecondBatch.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Failed").Should().Be(1); } [Test] public void ThenTwoErrorLogRecords() { _helperSecondBatch.RowCount("procfwk.ErrorLog").Should().Be(2); } #endregion [OneTimeTearDown] public void TearDown() { _helperFirstBatch?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/02-Parent/GivenBatchExecutionsForConcurrentBatchesWithSimpleFailureHandlingAndRestart.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Parent { public class GivenBatchExecutionsForConcurrentBatchesWithSimpleFailureHandlingAndRestart { private ParentHelper _helperFirstBatch; private ParentHelper _helperSecondBatch; private ParentHelper _helperSecondBatchRestart; private ParentHelper _helperThirdBatch; [OneTimeSetUp] public async Task WhenPipelineRun() { //first _helperFirstBatch = new ParentHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .WithSPNInDatabase("FrameworkFactory") .WithEmptyExecutionTables() .WithBatchExecutionHandling() .WithStagesEnabled() .WithPipelinesEnabled() .WithParameter("BatchName", "Hourly"); _helperSecondBatch = new ParentHelper() .WithSimulatedError() .WithFailureHandling("Simple") .WithParameter("BatchName", "Daily"); var firstBatch = _helperFirstBatch.RunPipeline(); var secondBatch = _helperSecondBatch.RunPipeline(); await Task.WhenAll(firstBatch, secondBatch); //then _helperSecondBatchRestart = new ParentHelper() .WithoutSimulatedError() .WithParameter("BatchName", "Daily"); _helperThirdBatch = new ParentHelper() .WithParameter("BatchName", "Hourly"); var secondBatchRestart = _helperSecondBatchRestart.RunPipeline(10000); //10 second delay var thirdBatch = _helperThirdBatch.RunPipeline(); await Task.WhenAll(secondBatchRestart, thirdBatch); } #region Integration tests //batch one - hourly: [Test] public void ThenFirstBatchPipelineOutcomeIsSucceeded() { _helperFirstBatch.RunOutcome.Should().Be("Succeeded"); } //batch two - daily: [Test] public void ThenSecondBatchPipelineOutcomeIsFailed() { _helperSecondBatch.RunOutcome.Should().Be("Failed"); } //batch two restart - daily: [Test] public void ThenSecondBatchRestartPipelineOutcomeIsSucceeded() { _helperSecondBatchRestart.RunOutcome.Should().Be("Succeeded"); } //batch three - hourly: [Test] public void ThenThirdBatchPipelineOutcomeIsSucceeded() { _helperThirdBatch.RunOutcome.Should().Be("Succeeded"); } #endregion #region Functional Tests [Test] public void ThenThreeBatchExecutionSuccessRecords() { _helperThirdBatch.RowCount("procfwk.BatchExecution", where: "BatchStatus", equals: "Success").Should().Be(3); } [Test] public void ThenNineteenExecutionLogSuccessRecords() { _helperThirdBatch.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Success").Should().Be(19); } [Test] public void ThenOneExecutionLogRecord() { _helperThirdBatch.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Failed").Should().Be(1); } [Test] public void ThenTwoErrorLogRecords() { _helperThirdBatch.RowCount("procfwk.ErrorLog").Should().Be(2); } [Test] public void ThenCurrentExecutionTableIsEmpty() { _helperThirdBatch.RowCount("procfwk.CurrentExecution").Should().Be(0); } #endregion [OneTimeTearDown] public void TearDown() { _helperThirdBatch?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/02-Parent/GivenBatchExecutionsForConcurrentBatchesWithSimpleFailureHandlingAndRestartOveride.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Parent { public class GivenBatchExecutionsForConcurrentBatchesWithSimpleFailureHandlingAndRestartOveride { private ParentHelper _helperFirstBatch; private ParentHelper _helperSecondBatch; private ParentHelper _helperSecondBatchRestart; private ParentHelper _helperThirdBatch; [OneTimeSetUp] public async Task WhenPipelineRun() { //first _helperFirstBatch = new ParentHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .WithSPNInDatabase("FrameworkFactory") .WithEmptyExecutionTables() .WithBatchExecutionHandling() .WithStagesEnabled() .WithPipelinesEnabled() .WithParameter("BatchName", "Hourly"); _helperSecondBatch = new ParentHelper() .WithSimulatedError() .WithFailureHandling("Simple") .WithParameter("BatchName", "Daily"); var firstBatch = _helperFirstBatch.RunPipeline(); var secondBatch = _helperSecondBatch.RunPipeline(); await Task.WhenAll(firstBatch, secondBatch); //then _helperSecondBatchRestart = new ParentHelper() .WithoutSimulatedError() .WithOverideRestart(true) .WithParameter("BatchName", "Daily"); _helperThirdBatch = new ParentHelper() .WithParameter("BatchName", "Hourly"); var secondBatchRestart = _helperSecondBatchRestart.RunPipeline(10000); //10 second delay var thirdBatch = _helperThirdBatch.RunPipeline(); await Task.WhenAll(secondBatchRestart, thirdBatch); } #region Integration tests //batch one - hourly: [Test] public void ThenFirstBatchPipelineOutcomeIsSucceeded() { _helperFirstBatch.RunOutcome.Should().Be("Succeeded"); } //batch two - daily: [Test] public void ThenSecondBatchPipelineOutcomeIsFailed() { _helperSecondBatch.RunOutcome.Should().Be("Failed"); } //batch two restart - daily: [Test] public void ThenSecondBatchRestartPipelineOutcomeIsSucceeded() { _helperSecondBatchRestart.RunOutcome.Should().Be("Succeeded"); } //batch three - hourly: [Test] public void ThenThirdBatchPipelineOutcomeIsSucceeded() { _helperThirdBatch.RunOutcome.Should().Be("Succeeded"); } #endregion #region Functional Tests [Test] public void ThenThreeBatchExecutionSuccessRecords() { _helperThirdBatch.RowCount("procfwk.BatchExecution", where: "BatchStatus", equals: "Success").Should().Be(3); } [Test] public void ThenOneBatchExecutionAbandonedRecord() { _helperThirdBatch.RowCount("procfwk.BatchExecution", where: "BatchStatus", equals: "Abandoned").Should().Be(1); } [Test] public void ThenTwentyTwoExecutionLogSuccessRecords() { _helperThirdBatch.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Success").Should().Be(22); } [Test] public void ThenOneExecutionLogRecord() { _helperThirdBatch.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Failed").Should().Be(2); } [Test] public void ThenSevenExecutionsBlocked() { _helperThirdBatch.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Blocked").Should().Be(7); } [Test] public void ThenTwoErrorLogRecords() { _helperThirdBatch.RowCount("procfwk.ErrorLog").Should().Be(2); } [Test] public void ThenCurrentExecutionTableIsEmpty() { _helperThirdBatch.RowCount("procfwk.CurrentExecution").Should().Be(0); } #endregion [OneTimeTearDown] public void TearDown() { _helperThirdBatch?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/02-Parent/GivenBatchExecutionsForSingleBatch.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Parent { public class GivenBatchExecutionsForSingleBatch { private ParentHelper _helper; [OneTimeSetUp] public async Task WhenPipelineRun() { _helper = new ParentHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .WithSPNInDatabase("FrameworkFactory") .WithEmptyExecutionTables() .WithBatchExecutionHandling() .WithStagesEnabled() .WithPipelinesEnabled() .WithParameter("BatchName", "Hourly"); await _helper.RunPipeline(); } #region Functional tests [Test] public void ThenPipelineOutcomeIsSucceeded() { _helper.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenCurrentExecutionTableIsEmpty() { _helper.RowCount("procfwk.CurrentExecution").Should().Be(0); } [Test] public void ThenFourExecutionLogSuccessRecords() { _helper.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Success").Should().Be(4); } [Test] public void ThenOneBatchExecutionSuccessRecord() { _helper.RowCount("procfwk.BatchExecution", where: "BatchStatus", equals: "Success").Should().Be(1); } #endregion [OneTimeTearDown] public void TearDown() { _helper?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/02-Parent/GivenCancelledWorkerAndRestart.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Parent { public class GivenCancelledWorkerAndRestart { private ParentHelper _helperFirstRun; private ParentHelper _helperRestartRun; [OneTimeSetUp] public async Task WhenPipelineRun() { _helperFirstRun = new ParentHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .WithSPNInDatabase("FrameworkFactory") .WithEmptyExecutionTables() .WithoutPrecursorObject() //done to ensure 2min waits are used, not example precursor waits .With2MinWaitsOnWorkers() //to ensure the cancel call has enough time .WithCancelledWorkersBlock(true) .WithFailureHandling("Simple"); var runOrchestrator = _helperFirstRun.RunPipeline(); var cancelWorker = _helperFirstRun.CancelAnyWorkerPipeline(); await Task.WhenAll(runOrchestrator, cancelWorker); _helperRestartRun = new ParentHelper(); await _helperRestartRun.RunPipeline(); } #region Integration tests [Test] public void ThenPipelineOutcomeIsSucceeded() { _helperRestartRun.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenOneExecutionLogRecord() { _helperRestartRun.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Cancelled").Should().Be(1); } [Test] public void ThenElevenExecutionsSucceeded() { _helperRestartRun.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Success").Should().Be(11); } #endregion [OneTimeTearDown] public void TearDown() { _helperRestartRun?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/02-Parent/GivenCancelledWorkerInOneExecutionStage.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Parent { public class GivenCancelledWorkerInOneExecutionStage { private ParentHelper _helper; [OneTimeSetUp] public async Task WhenPipelineRun() { _helper = new ParentHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .WithSPNInDatabase("FrameworkFactory") .WithEmptyExecutionTables() .WithoutPrecursorObject() //done to ensure 2min waits are used, not example precursor waits .With2MinWaitsOnWorkers() //to ensure the cancel call has enough time .WithCancelledWorkersBlock(false) .WithOnlyStageOneEnabled(); var runOrchestrator = _helper.RunPipeline(); var cancelWorker = _helper.CancelAnyWorkerPipeline(); //any worker await Task.WhenAll(runOrchestrator, cancelWorker); } #region Integration tests [Test] public void ThenPipelineOutcomeIsFailed() { _helper.RunOutcome.Should().Be("Failed"); } [Test] public void ThenOneExecutionsCancelled() { _helper.RowCount("procfwk.CurrentExecution", where: "PipelineStatus", equals: "Cancelled").Should().Be(1); } [Test] public void ThenOneExecutionLogRecord() { _helper.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Cancelled").Should().Be(1); } [Test] public void ThenThreeExecutionsSucceeded() { _helper.RowCount("procfwk.CurrentExecution", where: "PipelineStatus", equals: "Success").Should().Be(3); } #endregion [OneTimeTearDown] public void TearDown() { _helper?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/02-Parent/GivenCancelledWorkerThatBlocks.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Parent { public class GivenCancelledWorkerThatBlocks { private ParentHelper _helper; [OneTimeSetUp] public async Task WhenPipelineRun() { _helper = new ParentHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .WithSPNInDatabase("FrameworkFactory") .WithEmptyExecutionTables() .WithoutPrecursorObject() //done to ensure 2min waits are used, not example precursor waits .With2MinWaitsOnWorkers() //to ensure the cancel call has enough time .WithCancelledWorkersBlock(true) .WithFailureHandling("Simple"); var runOrchestrator = _helper.RunPipeline(); var cancelWorker = _helper.CancelIntentionalErrorWorkerPipeline(); //specific worker await Task.WhenAll(runOrchestrator, cancelWorker); } #region Integration tests [Test] public void ThenPipelineOutcomeIsFailed() { _helper.RunOutcome.Should().Be("Failed"); } [Test] public void ThenOneExecutionsCancelled() { _helper.RowCount("procfwk.CurrentExecution", where: "PipelineStatus", equals: "Cancelled").Should().Be(1); } [Test] public void ThenOneExecutionLogRecord() { _helper.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Cancelled").Should().Be(1); } [Test] public void ThenThreeExecutionsSucceeded() { _helper.RowCount("procfwk.CurrentExecution", where: "PipelineStatus", equals: "Success").Should().Be(3); } [Test] public void ThenSevenExecutionsBlocked() { _helper.RowCount("procfwk.CurrentExecution", where: "PipelineStatus", equals: "Blocked").Should().Be(7); } #endregion [OneTimeTearDown] public void TearDown() { _helper?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/02-Parent/GivenCancelledWorkerThatDoesntBlock.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Parent { public class GivenCancelledWorkerThatDoesntBlock { private ParentHelper _helper; [OneTimeSetUp] public async Task WhenPipelineRun() { _helper = new ParentHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .WithSPNInDatabase("FrameworkFactory") .WithEmptyExecutionTables() .WithoutPrecursorObject() //done to ensure 2min waits are used, not example precursor waits .With2MinWaitsOnWorkers() //to ensure the cancel call has enough time .WithCancelledWorkersBlock(false) .WithFailureHandling("Simple"); var runOrchestrator = _helper.RunPipeline(); var cancelWorker = _helper.CancelIntentionalErrorWorkerPipeline(); //specific worker await Task.WhenAll(runOrchestrator, cancelWorker); } #region Integration tests [Test] public void ThenPipelineOutcomeIsFailed() { _helper.RunOutcome.Should().Be("Failed"); } [Test] public void ThenOneExecutionsCancelled() { _helper.RowCount("procfwk.CurrentExecution", where: "PipelineStatus", equals: "Cancelled").Should().Be(1); } [Test] public void ThenOneExecutionLogRecord() { _helper.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Cancelled").Should().Be(1); } [Test] public void ThenTenExecutionsSucceeded() { _helper.RowCount("procfwk.CurrentExecution", where: "PipelineStatus", equals: "Success").Should().Be(10); } #endregion [OneTimeTearDown] public void TearDown() { _helper?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/02-Parent/GivenCleanUpForCancelledWorkerAndRestart.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Parent { public class GivenCleanUpForCancelledWorkerAndRestart { private ParentHelper _helperFirstRun; private ParentHelper _helperRestartRun; [OneTimeSetUp] public async Task WhenPipelineRun() { _helperFirstRun = new ParentHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .WithSPNInDatabase("FrameworkFactory") .WithEmptyExecutionTables() .WithoutPrecursorObject() //done to ensure 2min waits are used, not example precursor waits .With2MinWaitsOnWorkers() //to ensure the cancel call has enough time .WithCancelledWorkersBlock(true) .WithFailureHandling("Simple"); var runOrchestrator = _helperFirstRun.RunPipeline(); var cancelWorker = _helperFirstRun.CancelAnyWorkerPipeline(); await Task.WhenAll(runOrchestrator, cancelWorker); _helperRestartRun = new ParentHelper() .WithPrecursorObject() .WithRunningPipelineStatusInPlaceOf("Cancelled"); await _helperRestartRun.RunPipeline(); } #region Integration tests [Test] public void ThenPipelineOutcomeIsSucceeded() { _helperRestartRun.RunOutcome.Should().Be("Succeeded"); } [Test] public async Task ThenActivityShouldReturnOneRowForCleanUp() { var count = await _helperRestartRun.GetActivityOutput("Check Previous Execution", "$.count"); int.Parse(count).Should().Be(1); } [Test] public void ThenOneExecutionLogRecord() { _helperRestartRun.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Cancelled").Should().Be(1); } [Test] public void ThenElevenExecutionsSucceeded() { _helperRestartRun.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Success").Should().Be(11); } #endregion [OneTimeTearDown] public void TearDown() { _helperRestartRun?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/02-Parent/GivenCleanUpForSuccessfulWorkersAndRestart.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Parent { public class GivenCleanUpForSuccessfulWorkersAndRestart { private ParentHelper _helperFirstRun; private ParentHelper _helperRestartRun; [OneTimeSetUp] public async Task WhenPipelineRun() { _helperFirstRun = new ParentHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .WithSPNInDatabase("FrameworkFactory") .WithEmptyExecutionTables() .WithoutPrecursorObject() //done to ensure 2min waits are used, not example precursor waits .With2MinWaitsOnWorkers() //to ensure the cancel call has enough time .WithCancelledWorkersBlock(true) .WithFailureHandling("Simple"); var runOrchestrator = _helperFirstRun.RunPipeline(); var cancelWorker = _helperFirstRun.CancelAnyWorkerPipeline(); await Task.WhenAll(runOrchestrator, cancelWorker); _helperRestartRun = new ParentHelper() .WithPrecursorObject() .WithRunningPipelineStatusInPlaceOf("Success"); await _helperRestartRun.RunPipeline(); } [Test] public void ThenPipelineOutcomeIsSucceeded() { _helperRestartRun.RunOutcome.Should().Be("Succeeded"); } [Test] public async Task ThenActivityShouldReturnThreeRowsForCleanUp() { var count = await _helperRestartRun.GetActivityOutput("Check Previous Execution", "$.count"); int.Parse(count).Should().Be(3); } [Test] public void ThenOneExecutionLogRecord() { _helperRestartRun.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Cancelled").Should().Be(1); } [Test] public void ThenElevenExecutionsSucceeded() { _helperRestartRun.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Success").Should().Be(11); } [OneTimeTearDown] public void TearDown() { _helperRestartRun?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/02-Parent/GivenDependencyChainFailureHandling.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Parent { public class GivenDependencyChainFailureHandling { private ParentHelper _helper; [OneTimeSetUp] public async Task WhenPipelineRun() { _helper = new ParentHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .WithSPNInDatabase("FrameworkFactory") .WithEmptyExecutionTables() .WithSimulatedError() .WithFailureHandling("DependencyChain"); await _helper.RunPipeline(); } #region Functional tests [Test, Order(1)] public void ThenPipelineOutcomeIsFailed() { _helper.RunOutcome.Should().Be("Failed"); } [Test, Order(2)] public void ThenSixExecutionsSucceeded() { _helper.RowCount("procfwk.CurrentExecution", where: "PipelineStatus", equals: "Success").Should().Be(6); } [Test, Order(3)] public void ThenOneExecutionFailed() { _helper.RowCount("procfwk.CurrentExecution", where: "PipelineStatus", equals: "Failed").Should().Be(1); } [Test, Order(4)] public void ThenFourExecutionsBlocked() { _helper.RowCount("procfwk.CurrentExecution", where: "PipelineStatus", equals: "Blocked").Should().Be(4); } [Test, Order(5)] public void ThenOneExecutionLogRecord() { _helper.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Failed").Should().Be(1); } [Test, Order(6)] public void ThenTwoErrorLogRecords() { _helper.RowCount("procfwk.ErrorLog").Should().Be(2); } #endregion [OneTimeTearDown] public void TearDown() { _helper?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/02-Parent/GivenDependencyChainFailureHandlingAndRestart.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Parent { public class GivenDependencyChainFailureHandlingAndRestart { private ParentHelper _helperFirstRun; private ParentHelper _helperRestartRun; [OneTimeSetUp] public async Task WhenPipelineRun() { _helperFirstRun = new ParentHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .WithSPNInDatabase("FrameworkFactory") .WithEmptyExecutionTables() .WithSimulatedError() .WithFailureHandling("DependencyChain"); await _helperFirstRun.RunPipeline(); _helperRestartRun = new ParentHelper() .WithoutSimulatedError(); await _helperRestartRun.RunPipeline(); } #region Functional tests [Test] public void ThenPipelineOutcomeIsSucceeded() { _helperRestartRun.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenOneExecutionLogFailedRecord() { _helperRestartRun.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Failed").Should().Be(1); } [Test] public void ThenElevenExecutionLogSuccessRecord() { _helperRestartRun.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Success").Should().Be(11); } [Test] public void ThenTwoErrorLogRecords() { _helperRestartRun.RowCount("procfwk.ErrorLog").Should().Be(2); } #endregion [OneTimeTearDown] public void TearDown() { _helperFirstRun?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/02-Parent/GivenDisabledBatches.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Parent { public class GivenDisabledBatches { private ParentHelper _helper; [OneTimeSetUp] public async Task WhenPipelineRun() { _helper = new ParentHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .WithSPNInDatabase("FrameworkFactory") .WithEmptyExecutionTables() .WithBatchExecutionHandling() .WithBatchesDisabled(); await _helper.RunPipeline(); } #region Integration tests [Test] public void ThenPipelineOutcomeIsFailed() { _helper.RunOutcome.Should().Be("Failed"); } #endregion [OneTimeTearDown] public void TearDown() { _helper?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/02-Parent/GivenDisabledPipelines.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Parent { class GivenDisabledPipelines { private ParentHelper _helper; [OneTimeSetUp] public async Task WhenPipelineRun() { _helper = new ParentHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .WithSPNInDatabase("FrameworkFactory") .WithEmptyExecutionTables() .WithPipelinesDisabled(); await _helper.RunPipeline(); } #region Integration tests [Test, Order(1)] public void ThenPipelineOutcomeIsFailed() { _helper.RunOutcome.Should().Be("Failed"); } #endregion [OneTimeTearDown] public void TearDown() { _helper?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/02-Parent/GivenDisabledStages.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Parent { public class GivenDisabledStages { private ParentHelper _helper; [OneTimeSetUp] public async Task WhenPipelineRun() { _helper = new ParentHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .WithSPNInDatabase("FrameworkFactory") .WithEmptyExecutionTables() .WithStagesDisabled(); await _helper.RunPipeline(); } #region Integration tests [Test, Order(1)] public void ThenPipelineOutcomeIsFailed() { _helper.RunOutcome.Should().Be("Failed"); } #endregion [OneTimeTearDown] public void TearDown() { _helper?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/02-Parent/GivenNoErrorsAndSPNStoredInDatabase.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Parent { public class GivenNoErrorsAndSPNStoredInDatabase { private ParentHelper _helper; [OneTimeSetUp] public async Task WhenPipelineRun() { _helper = new ParentHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .WithSPNInDatabase("FrameworkFactory") .WithEmptyExecutionTables() .WithoutSimulatedError() .WithFailureHandling("Simple"); await _helper.RunPipeline(); } #region Integration tests [Test] public void ThenPipelineOutcomeIsSucceeded() { _helper.RunOutcome.Should().Be("Succeeded"); } #endregion #region Functional tests [Test] public void ThenCurrentExecutionTableIsEmpty() { _helper.RowCount("procfwk.CurrentExecution").Should().Be(0); } [Test] public void ThenElevenExecutionLogRecords() { _helper.RowCount("procfwk.ExecutionLog").Should().Be(11); } #endregion [OneTimeTearDown] public void TearDown() { _helper?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/02-Parent/GivenNoErrorsAndSPNStoredInKeyVault.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Parent { public class GivenNoErrorsAndSPNStoredInKeyVault { private ParentHelper _helper; [OneTimeSetUp] public async Task WhenPipelineRun() { _helper = new ParentHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .WithSPNInKeyVault("FrameworkFactory") .WithEmptyExecutionTables() .WithoutSimulatedError() .WithFailureHandling("Simple"); await _helper.RunPipeline(); } #region Integration tests [Test, Order(1)] public void ThenPipelineOutcomeIsSucceeded() { _helper.RunOutcome.Should().Be("Succeeded"); } #endregion #region Functional tests [Test, Order(2)] public void ThenCurrentExecutionTableIsEmpty() { _helper.RowCount("procfwk.CurrentExecution").Should().Be(0); } [Test, Order(3)] public void ThenElevenExecutionLogRecords() { _helper.RowCount("procfwk.ExecutionLog").Should().Be(11); } #endregion [OneTimeTearDown] public void TearDown() { _helper?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/02-Parent/GivenNoFailureHandling.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Parent { public class GivenNoFailureHandling { private ParentHelper _helper; [OneTimeSetUp] public async Task WhenPipelineRun() { _helper = new ParentHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .WithSPNInDatabase("FrameworkFactory") .WithEmptyExecutionTables() .WithSimulatedError() .WithFailureHandling("None"); await _helper.RunPipeline(); } #region Functional tests [Test, Order(1)] public void ThenPipelineOutcomeIsFailed() { _helper.RunOutcome.Should().Be("Failed"); } [Test, Order(2)] public void ThenTenExecutionsSucceeded() { _helper.RowCount("procfwk.CurrentExecution", where: "PipelineStatus", equals: "Success").Should().Be(10); } [Test, Order(3)] public void ThenOneExecutionFailed() { _helper.RowCount("procfwk.CurrentExecution", where: "PipelineStatus", equals: "Failed").Should().Be(1); } [Test, Order(4)] public void ThenOneExecutionLogRecord() { _helper.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Failed").Should().Be(1); } [Test, Order(5)] public void ThenTwoErrorLogRecords() { _helper.RowCount("procfwk.ErrorLog").Should().Be(2); } #endregion [OneTimeTearDown] public void TearDown() { _helper?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/02-Parent/GivenNoPipelineParameters.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Parent { class GivenNoPipelineParameters { private ParentHelper _helper; [OneTimeSetUp] public async Task WhenPipelineRun() { _helper = new ParentHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .WithSPNInDatabase("FrameworkFactory") .WithEmptyExecutionTables() .WithoutSimulatedError() .WithFailureHandling("Simple"); ; await _helper.RunPipeline(); } #region Integration tests [Test] public void ThenPipelineOutcomeIsSucceeded() { _helper.RunOutcome.Should().Be("Succeeded"); } #endregion #region Functional tests #endregion [OneTimeTearDown] public void TearDown() { _helper?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/02-Parent/GivenOneExecutionStage.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Parent { class GivenOneExecutionStage { private ParentHelper _helper; [OneTimeSetUp] public async Task WhenPipelineRun() { _helper = new ParentHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .WithSPNInDatabase("FrameworkFactory") .WithEmptyExecutionTables() .WithoutSimulatedError() .WithFailureHandling("Simple") .WithSingleExecutionStage(); await _helper.RunPipeline(); } #region Integration tests [Test] public void ThenPipelineOutcomeIsSucceeded() { _helper.RunOutcome.Should().Be("Succeeded"); } #endregion [OneTimeTearDown] public void TearDown() { _helper?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/02-Parent/GivenSimpleFailureHandling.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Parent { public class GivenSimpleFailureHandling { private ParentHelper _helper; [OneTimeSetUp] public async Task WhenPipelineRun() { _helper = new ParentHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .WithSPNInDatabase("FrameworkFactory") .WithEmptyExecutionTables() .WithSimulatedError() .WithFailureHandling("Simple"); await _helper.RunPipeline(); } #region Functional tests [Test, Order(1)] public void ThenPipelineOutcomeIsFailed() { _helper.RunOutcome.Should().Be("Failed"); } [Test, Order(2)] public void ThenThreeExecutionsSucceeded() { _helper.RowCount("procfwk.CurrentExecution", where: "PipelineStatus", equals: "Success").Should().Be(3); } [Test, Order(3)] public void ThenOneExecutionFailed() { _helper.RowCount("procfwk.CurrentExecution", where: "PipelineStatus", equals: "Failed").Should().Be(1); } [Test, Order(4)] public void ThenSevenExecutionsBlocked() { _helper.RowCount("procfwk.CurrentExecution", where: "PipelineStatus", equals: "Blocked").Should().Be(7); } [Test, Order(5)] public void ThenOneExecutionLogRecord() { _helper.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Failed").Should().Be(1); } [Test, Order(6)] public void ThenTwoErrorLogRecords() { _helper.RowCount("procfwk.ErrorLog").Should().Be(2); } #endregion [OneTimeTearDown] public void TearDown() { _helper?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/02-Parent/GivenSimpleFailureHandlingAndRestart.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Parent { public class GivenSimpleFailureHandlingAndRestart { private ParentHelper _helperFirstRun; private ParentHelper _helperRestartRun; [OneTimeSetUp] public async Task WhenPipelineRun() { _helperFirstRun = new ParentHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .WithSPNInDatabase("FrameworkFactory") .WithEmptyExecutionTables() .WithSimulatedError() .WithFailureHandling("Simple"); await _helperFirstRun.RunPipeline(); _helperRestartRun = new ParentHelper() .WithoutSimulatedError(); await _helperRestartRun.RunPipeline(); } #region Functional tests [Test] public void ThenPipelineOutcomeIsSucceeded() { _helperRestartRun.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenOneExecutionLogFailedRecord() { _helperRestartRun.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Failed").Should().Be(1); } [Test] public void ThenElevenExecutionLogSuccessRecord() { _helperRestartRun.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Success").Should().Be(11); } [Test] public void ThenTwoErrorLogRecords() { _helperRestartRun.RowCount("procfwk.ErrorLog").Should().Be(2); } #endregion [OneTimeTearDown] public void TearDown() { _helperRestartRun?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/02-Parent/GivenSimpleFailureHandlingAndRestartOveride.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Parent { public class GivenSimpleFailureHandlingAndRestartOveride { private ParentHelper _helperFirstRun; private ParentHelper _helperRestartRun; [OneTimeSetUp] public async Task WhenPipelineRun() { _helperFirstRun = new ParentHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .WithSPNInDatabase("FrameworkFactory") .WithEmptyExecutionTables() .WithSimulatedError() .WithFailureHandling("Simple"); await _helperFirstRun.RunPipeline(); _helperRestartRun = new ParentHelper() .WithoutSimulatedError() .WithOverideRestart(true); await _helperRestartRun.RunPipeline(); } #region Functional tests [Test] public void ThenPipelineOutcomeIsSucceeded() { _helperRestartRun.RunOutcome.Should().Be("Succeeded"); } [Test] public void ThenOneExecutionLogFailedRecord() { _helperRestartRun.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Failed").Should().Be(2); } [Test] public void ThenFourthteenExecutionLogSuccessRecord() { _helperRestartRun.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Success").Should().Be(14); } [Test] public void ThenSevenExecutionsBlocked() { _helperRestartRun.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Blocked").Should().Be(7); } [Test] public void ThenTwoErrorLogRecords() { _helperRestartRun.RowCount("procfwk.ErrorLog").Should().Be(2); } #endregion [OneTimeTearDown] public void TearDown() { _helperRestartRun?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/02-Parent/ParentHelper.cs ================================================ using FactoryTesting.Helpers; using System; using System.Data.SqlClient; using System.Threading; using System.Threading.Tasks; using System.Xml; namespace FactoryTesting.Pipelines.Parent { class ParentHelper : CoverageHelper { public async Task RunPipeline() { await RunPipeline("02-Parent"); } public async Task RunPipeline(int fakeDelayMilliseconds) { Thread.Sleep(fakeDelayMilliseconds); await RunPipeline("02-Parent"); } public async Task CancelAnyWorkerPipeline() { await CancelRunningPipeline(GetWorkerRunId(), GetSetting("WorkersDataFactoryName")); } public async Task CancelIntentionalErrorWorkerPipeline() { await CancelRunningPipeline(GetWorkerRunId("Intentional Error"), GetSetting("WorkersDataFactoryName")); } public virtual Task RunAsync() { return Task.CompletedTask; } public ParentHelper WithTenantAndSubscriptionIds() { AddTenantAndSubscription(); return this; } public ParentHelper WithSPNInDatabase(string workerFactoryName) { AddWorkerSPNStoredInDatabase(workerFactoryName); return this; } public ParentHelper WithSPNInKeyVault(string workerFactoryName) { AddWorkerSPNStoredInKeyVault(workerFactoryName); return this; } public ParentHelper WithBasicMetadata() { AddBasicMetadata(); return this; } public ParentHelper WithEmptyExecutionTables() { WithEmptyTable("procfwk.CurrentExecution"); WithEmptyTable("procfwk.ExecutionLog"); WithEmptyTable("procfwk.ErrorLog"); return this; } public ParentHelper WithRunningPipelineStatusInPlaceOf(string statusToOveride) { SetFalsePipelineStatus("Running", "PipelineStatus", statusToOveride); return this; } public ParentHelper WithSimulatedError() { SimulateError(true); return this; } public ParentHelper WithoutSimulatedError() { SimulateError(false); return this; } public ParentHelper WithBatchesDisabled() { EnableDisableMetadata("Batches", false); return this; } public ParentHelper WithStagesDisabled() { EnableDisableMetadata("Stages", false); return this; } public ParentHelper WithStagesEnabled() { EnableDisableMetadata("Stages", true); return this; } public ParentHelper WithOnlyStageOneEnabled() { EnableDisableMetadata("Stages", false, "StageId", "2"); EnableDisableMetadata("Stages", false, "StageId", "3"); EnableDisableMetadata("Stages", false, "StageId", "4"); EnableDisableMetadata("Stages", false, "StageId", "5"); return this; } public ParentHelper WithPipelinesDisabled() { EnableDisableMetadata("Pipelines", false); return this; } public ParentHelper WithPipelinesEnabled() { EnableDisableMetadata("Pipelines", true); return this; } public ParentHelper With2MinWaitsOnWorkers() { SetParameterValue("120", "ParameterName", "WaitTime"); return this; } public ParentHelper WithBatchExecutionHandling() { ExecuteNonQuery(@$"UPDATE [procfwk].[Properties] SET [PropertyValue] = '1' WHERE [PropertyName] = 'UseExecutionBatches'"); return this; } public ParentHelper WithoutBatchExecutionHandling() { ExecuteNonQuery(@$"UPDATE [procfwk].[Properties] SET [PropertyValue] = '0' WHERE [PropertyName] = 'UseExecutionBatches'"); return this; } public ParentHelper WithFailureHandling(string mode) { ExecuteNonQuery(@$"UPDATE [procfwk].[Properties] SET [PropertyValue] = '{mode}' WHERE [PropertyName] = 'FailureHandling'"); return this; } public ParentHelper WithCancelledWorkersBlock(bool mode) { string modeString = mode ? "1" : "0"; ExecuteNonQuery(@$"UPDATE [procfwk].[Properties] SET [PropertyValue] = '{modeString}' WHERE [PropertyName] = 'CancelledWorkerResultBlocks'"); return this; } public ParentHelper WithOverideRestart(bool mode) { string modeString = mode ? "1" : "0"; ExecuteNonQuery(@$"UPDATE [procfwk].[Properties] SET [PropertyValue] = '{modeString}' WHERE [PropertyName] = 'OverideRestart'"); return this; } public ParentHelper WithoutPrecursorObject() { ExecuteNonQuery(@$"UPDATE [procfwk].[Properties] SET [PropertyValue] = '[dbo].[none]' WHERE [PropertyName] = 'ExecutionPrecursorProc'"); return this; } public ParentHelper WithPrecursorObject() { ExecuteNonQuery(@$"UPDATE [procfwk].[Properties] SET [PropertyValue] = '[dbo].[ExampleCustomExecutionPrecursor]' WHERE [PropertyName] = 'ExecutionPrecursorProc'"); return this; } public ParentHelper WithSingleExecutionStage() { ExecuteNonQuery("UPDATE [procfwk].[Pipelines] SET [StageId] = 1"); return this; } public ParentHelper WithCustom() { ExecuteStoredProcedure("[dbo].[PaulTemp]", null); return this; } private ParentHelper SetFalsePipelineStatus(string falseStatus, string where, string equals) { ExecuteNonQuery($"UPDATE [procfwk].[CurrentExecution] SET [PipelineStatus] = '{falseStatus}' WHERE {where} = '{equals.Replace("'", "''")}'"); return this; } private string GetWorkerRunId(string pipelineName = null) { string PipelineRunId; using (var cmd = new SqlCommand("procfwkTesting.GetRunIdWhenAvailable", _conn)) { cmd.CommandTimeout = 600; cmd.CommandType = System.Data.CommandType.StoredProcedure; if (pipelineName != null) cmd.Parameters.Add(new SqlParameter("@PipelineName", pipelineName)); using var reader = cmd.ExecuteReader(); reader.Read(); PipelineRunId = reader.GetString(0).ToLower(); } return PipelineRunId; } private void EnableDisableMetadata(string table, bool state) { string paramValue = state ? "true" : "false"; ExecuteNonQuery(@$"UPDATE [procfwk].[{table}] SET [Enabled] = '{paramValue}'"); } private void EnableDisableMetadata(string table, bool state, string where, string equals) { string paramValue = state ? "true" : "false"; ExecuteNonQuery(@$"UPDATE [procfwk].[{table}] SET [Enabled] = '{paramValue}' WHERE {where} = '{equals.Replace("'", "''")}'"); } private void SetParameterValue(string value, string where, string equals) { string sqlStatement = @$"UPDATE [procfwk].[PipelineParameters] SET [ParameterValue] = '{value.Replace("'", "''")}' WHERE {where} = '{equals.Replace("'", "''")}'"; ExecuteNonQuery(sqlStatement); } private void SimulateError(bool simulate) { string paramValue = simulate ? "true" : "false"; ExecuteNonQuery(@$"UPDATE pp SET [ParameterValue] = '{paramValue}' FROM [procfwk].[PipelineParameters] pp INNER JOIN [procfwk].[Pipelines] p ON pp.[PipelineId] = p.[PipelineId] WHERE p.[PipelineName] = 'Intentional Error' AND pp.[ParameterName] = 'RaiseErrors'"); } public override void TearDown() { base.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/Utilities/GivenCheckForRunningPipeline.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; using FactoryTesting.Helpers; namespace FactoryTesting.Pipelines.Utilities { public class GivenCheckForRunningPipeline { private UtilitiesHelper _helper; [OneTimeSetUp] public async Task WhenPipelineRun() { _helper = new UtilitiesHelper() .WithBasicMetadata() .WithTenantAndSubscriptionIds() .WithParameter("PipelineName", "Check For Running Pipeline"); await _helper.RunPipeline("Check For Running Pipeline"); } [Test] public void ThenPipelineOutcomeIsFailed() { _helper.RunOutcome.Should().Be("Failed"); } [Test] public async Task ThenActivityShouldReturnOneFilteredItemCount() { var filteredCount = await _helper.GetActivityOutput("Filter Running Pipelines", "$.FilteredItemsCount"); int.Parse(filteredCount).Should().Be(1); } [Test] public async Task ThenActivityShouldReturnMatchingSubscriptionId() { string subSetting = _helper.GetSetting("AZURE_SUBSCRIPTION_ID"); var subscriptionId = await _helper.GetActivityOutput("Set Subscription Id", "$.value"); subscriptionId.Should().Equals(subSetting.ToString()); } [Test] public async Task ThenActivityShouldReturnMatchingResourceGroup() { string rgSsetting = _helper.GetSetting("DataFactoryResourceGroup"); var resourceGroupName = await _helper.GetActivityOutput("Set Resource Group Name", "$.value"); resourceGroupName.Should().Equals(rgSsetting.ToString()); } [OneTimeTearDown] public void TearDown() { _helper?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/Utilities/GivenEmailSender.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Utilities { public class GivenEmailSender { private UtilitiesHelper _helper; [OneTimeSetUp] public async Task WhenPipelineRun() { _helper = new UtilitiesHelper() .WithParameter("Recipients", "paul@mrpaulandrew.com") .WithParameter("Subject", "NUnit Test") .WithParameter("Body", "NUnit Test"); ; await _helper.RunPipeline("Email Sender"); } [Test] public void ThenPipelineOutcomeIsSucceeded() { _helper.RunOutcome.Should().Be("Succeeded"); } [Test] public async Task ThenActivityShouldReturnEmailSentTrue() { var sentState = await _helper.GetActivityOutput("Send Email", "$.EmailSent"); bool.Parse(sentState).Should().Be(true); } [OneTimeTearDown] public void TearDown() { _helper?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/Utilities/GivenThrowException.cs ================================================ using FluentAssertions; using NUnit.Framework; using NUnit.Framework.Internal; using System.Threading.Tasks; namespace FactoryTesting.Pipelines.Utilities { public class GivenThrowException { private UtilitiesHelper _helper; [OneTimeSetUp] public async Task WhenPipelineRun() { _helper = new UtilitiesHelper() .WithParameter("Message","NUnit Test"); await _helper.RunPipeline("Throw Exception"); } [Test] public void ThenPipelineOutcomeIsFailed() { _helper.RunOutcome.Should().Be("Failed"); } [OneTimeTearDown] public void TearDown() { _helper?.TearDown(); } } } ================================================ FILE: FactoryTesting/Pipelines/Utilities/UtilitiesHelper.cs ================================================ using FactoryTesting.Helpers; using System; using System.Data.SqlClient; using System.Threading; using System.Threading.Tasks; using System.Xml; namespace FactoryTesting.Pipelines.Utilities { class UtilitiesHelper : CoverageHelper { public UtilitiesHelper WithBasicMetadata() { AddBasicMetadata(); return this; } public UtilitiesHelper WithTenantAndSubscriptionIds() { AddTenantAndSubscription(); return this; } public override void TearDown() { base.TearDown(); } } } ================================================ FILE: FactoryTesting/dev.runsettings ================================================ ================================================ FILE: FactoryTesting/multi.runsettings ================================================ ================================================ FILE: FactoryTesting/test.runsettings ================================================ ================================================ FILE: Functions/.vscode/extensions.json ================================================ { "recommendations": [ "ms-azuretools.vscode-azurefunctions", "ms-dotnettools.csharp" ] } ================================================ FILE: Functions/.vscode/launch.json ================================================ { "version": "0.2.0", "configurations": [ { "name": "Attach to .NET Functions", "type": "coreclr", "request": "attach", "processId": "${command:azureFunctions.pickProcess}" } ] } ================================================ FILE: Functions/.vscode/settings.json ================================================ { "azureFunctions.deploySubpath": "bin/Release/netcoreapp3.1/publish", "azureFunctions.projectLanguage": "C#", "azureFunctions.projectRuntime": "~3", "debug.internalConsoleOptions": "neverOpen", "azureFunctions.preDeployTask": "publish" } ================================================ FILE: Functions/.vscode/tasks.json ================================================ { "version": "2.0.0", "tasks": [ { "label": "clean", "command": "dotnet", "args": [ "clean", "/property:GenerateFullPaths=true", "/consoleloggerparameters:NoSummary" ], "type": "process", "problemMatcher": "$msCompile" }, { "label": "build", "command": "dotnet", "args": [ "build", "/property:GenerateFullPaths=true", "/consoleloggerparameters:NoSummary" ], "type": "process", "dependsOn": "clean", "group": { "kind": "build", "isDefault": true }, "problemMatcher": "$msCompile" }, { "label": "clean release", "command": "dotnet", "args": [ "clean", "--configuration", "Release", "/property:GenerateFullPaths=true", "/consoleloggerparameters:NoSummary" ], "type": "process", "problemMatcher": "$msCompile" }, { "label": "publish", "command": "dotnet", "args": [ "publish", "--configuration", "Release", "/property:GenerateFullPaths=true", "/consoleloggerparameters:NoSummary" ], "type": "process", "dependsOn": "clean release", "problemMatcher": "$msCompile" }, { "type": "func", "dependsOn": "build", "options": { "cwd": "${workspaceFolder}/bin/Debug/netcoreapp3.1" }, "command": "host start", "isBackground": true, "problemMatcher": "$func-watch" } ] } ================================================ FILE: Functions/Functions/CancelPipeline.cs ================================================ using System; using System.IO; using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc; using Microsoft.Azure.WebJobs; using Microsoft.Azure.WebJobs.Extensions.Http; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; using Newtonsoft.Json; using mrpaulandrew.azure.procfwk.Helpers; using mrpaulandrew.azure.procfwk.Services; namespace mrpaulandrew.azure.procfwk { public static class CancelPipeline { [FunctionName("CancelPipeline")] public static async Task Run( [HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequest httpRequest, ILogger logger) { logger.LogInformation("CancelPipeline Function triggered by HTTP request."); logger.LogInformation("Parsing body from request."); PipelineRunRequest request = await new BodyReader(httpRequest).GetRunRequestBodyAsync(); request.Validate(logger); using (var service = PipelineService.GetServiceForRequest(request, logger)) { PipelineRunStatus result = service.CancelPipeline(request); logger.LogInformation("CancelPipeline Function complete."); return new OkObjectResult(JsonConvert.SerializeObject(result)); } } } } ================================================ FILE: Functions/Functions/CheckPipelineStatus.cs ================================================ using System; using System.IO; using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc; using Microsoft.Azure.WebJobs; using Microsoft.Azure.WebJobs.Extensions.Http; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; using Newtonsoft.Json; using mrpaulandrew.azure.procfwk.Helpers; using mrpaulandrew.azure.procfwk.Services; namespace mrpaulandrew.azure.procfwk { public static class CheckPipelineStatus { [FunctionName("CheckPipelineStatus")] public static async Task Run( [HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest httpRequest, ILogger logger) { logger.LogInformation("CheckPipelineStatus Function triggered by HTTP request."); logger.LogInformation("Parsing body from request."); PipelineRunRequest request = await new BodyReader(httpRequest).GetRunRequestBodyAsync(); request.Validate(logger); using (var service = PipelineService.GetServiceForRequest(request, logger)) { PipelineRunStatus result = service.GetPipelineRunStatus(request); logger.LogInformation("CheckPipelineStatus Function complete."); return new OkObjectResult(JsonConvert.SerializeObject(result)); } } } } ================================================ FILE: Functions/Functions/ExecutePipeline.cs ================================================ using System; using System.IO; using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc; using Microsoft.Azure.WebJobs; using Microsoft.Azure.WebJobs.Extensions.Http; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; using Newtonsoft.Json; using mrpaulandrew.azure.procfwk.Helpers; using mrpaulandrew.azure.procfwk.Services; namespace mrpaulandrew.azure.procfwk { public static class ExecutePipeline { [FunctionName("ExecutePipeline")] public static async Task Run( [HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequest httpRequest, ILogger logger) { logger.LogInformation("ExecutePipeline Function triggered by HTTP request."); logger.LogInformation("Parsing body from request."); PipelineRequest request = await new BodyReader(httpRequest).GetRequestBodyAsync(); request.Validate(logger); using (var service = PipelineService.GetServiceForRequest(request, logger)) { PipelineRunStatus result = service.ExecutePipeline(request); logger.LogInformation("ExecutePipeline Function complete."); return new OkObjectResult(JsonConvert.SerializeObject(result)); } } } } ================================================ FILE: Functions/Functions/GetActivityErrors.cs ================================================ using System; using System.IO; using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc; using Microsoft.Azure.WebJobs; using Microsoft.Azure.WebJobs.Extensions.Http; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; using Newtonsoft.Json; using mrpaulandrew.azure.procfwk.Helpers; using mrpaulandrew.azure.procfwk.Services; namespace mrpaulandrew.azure.procfwk { public static class GetActivityErrors { [FunctionName("GetActivityErrors")] public static async Task Run( [HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest httpRequest, ILogger logger) { logger.LogInformation("GetActivityErrors Function triggered by HTTP request."); logger.LogInformation("Parsing body from request."); PipelineRunRequest request = await new BodyReader(httpRequest).GetRunRequestBodyAsync(); request.Validate(logger); using (var service = PipelineService.GetServiceForRequest(request, logger)) { PipelineErrorDetail result = service.GetPipelineRunActivityErrors(request); logger.LogInformation("GetActivityErrors Function complete."); return new OkObjectResult(JsonConvert.SerializeObject(result)); } } } } ================================================ FILE: Functions/Functions/SendEmail.cs ================================================ using System; using System.IO; using System.Threading.Tasks; using System.Net.Mail; using Microsoft.AspNetCore.Mvc; using Microsoft.Azure.WebJobs; using Microsoft.Azure.WebJobs.Extensions.Http; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using mrpaulandrew.azure.procfwk.Helpers; using System.Linq; namespace mrpaulandrew.azure.procfwk { public static class SendEmail { [FunctionName("SendEmail")] public static async Task Run( [HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequest req, ILogger log) { log.LogInformation("SendEmail Function triggered by HTTP request."); #region ParseRequestBody log.LogInformation("Parsing body from request."); string requestBody = await new StreamReader(req.Body).ReadToEndAsync(); dynamic data = JsonConvert.DeserializeObject(requestBody); string outputString = string.Empty; JObject outputJson; string toRecipients = data?.emailRecipients; string ccRecipients = data?.emailCcRecipients; string bccRecipients = data?.emailBccRecipients; string subject = data?.emailSubject; string message = data?.emailBody; string passedImportance = data?.emailImportance ?? ""; //Set normal importance if not provided. #endregion #region ValidateRequestBody //Check for minimum mailing values in request body if (subject == null || message == null) { log.LogInformation("Invalid body - Subject/Body."); outputString = "{ \"EmailSent\": false, \"Details\": \"Email subject or body values missing.\"}"; outputJson = JObject.Parse(outputString); return new BadRequestObjectResult(outputJson); } if ( (toRecipients == null && ccRecipients == null && bccRecipients == null) || (string.IsNullOrEmpty(toRecipients) && string.IsNullOrEmpty(ccRecipients) && string.IsNullOrEmpty(bccRecipients)) ) { log.LogInformation("Invalid body - To/CC/BCC."); outputString = "{ \"EmailSent\": false, \"Details\": \"No email recipients provided as To/CC/BCC.\"}"; outputJson = JObject.Parse(outputString); return new BadRequestObjectResult(outputJson); } #endregion //Create email client log.LogInformation("Creating smtp client."); using (var client = SMTPClient.CreateSMTPClient()) { #region CreateMail MailAddress from = new MailAddress(SMTPClient.FromEmail); MailMessage mail = new MailMessage { From = from, IsBodyHtml = true, Subject = subject, Body = message }; //Set mail importance if (passedImportance.ToUpper() == "HIGH") { mail.Priority = MailPriority.High; } else if (passedImportance.ToUpper() == "LOW") { mail.Priority = MailPriority.Low; } else { mail.Priority = MailPriority.Normal; } #endregion #region SetRecipients //to recipients if (!string.IsNullOrEmpty(toRecipients)) { var allRecipients = toRecipients.Split(new[] { "," }, StringSplitOptions.RemoveEmptyEntries); foreach (var toAddress in allRecipients) { mail.To.Add(toAddress); } log.LogInformation("To Recipients Added: " + allRecipients.Count().ToString()); } else { log.LogInformation("To Recipients Added: 0"); } //cc recipients if (!string.IsNullOrEmpty(ccRecipients)) { var allCcRecipients = ccRecipients.Split(new[] { "," }, StringSplitOptions.RemoveEmptyEntries); foreach (var ccAddress in allCcRecipients) { mail.CC.Add(ccAddress); } log.LogInformation("CC Recipients Added: " + allCcRecipients.Count().ToString()); } else { log.LogInformation("CC Recipients Added: 0"); } //bcc recipients if (!string.IsNullOrEmpty(bccRecipients)) { var allBccRecipients = bccRecipients.Split(new[] { "," }, StringSplitOptions.RemoveEmptyEntries); foreach (var bccAddress in allBccRecipients) { mail.Bcc.Add(bccAddress); } log.LogInformation("BCC Recipients Added: " + allBccRecipients.Count().ToString()); } else { log.LogInformation("BCC Recipients Added: 0"); } #endregion #region SendEmail try { log.LogInformation("Sending email."); client.Send(mail); outputString = "{ \"EmailSent\": true }"; log.LogInformation("Sent email."); } catch (SmtpException smtpEx) { outputString = "{ \"EmailSent\": false, \"Details\": \"SMTP exception caught and logged to error output.\"}"; log.LogError(smtpEx.Message); log.LogInformation("Message has not been sent. Check Azure Function Logs for more information."); } catch (Exception ex) { outputString = "{ \"EmailSent\": false, \"Details\": \"Other exception caught and logged to error output.\"}"; log.LogError(ex.Message); log.LogInformation("Message has not been sent. Check Azure Function Logs for more information."); } #endregion } outputJson = JObject.Parse(outputString); log.LogInformation("SendEmail Function complete."); return new OkObjectResult(outputJson); } } } ================================================ FILE: Functions/Functions/ValidatePipeline.cs ================================================ using System; using System.IO; using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc; using Microsoft.Azure.WebJobs; using Microsoft.Azure.WebJobs.Extensions.Http; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; using Newtonsoft.Json; using mrpaulandrew.azure.procfwk.Helpers; using mrpaulandrew.azure.procfwk.Services; namespace mrpaulandrew.azure.procfwk { public static class ValidatePipeline { [FunctionName("ValidatePipeline")] public static async Task Run( [HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest httpRequest, ILogger logger) { logger.LogInformation("ValidatePipeline Function triggered by HTTP request."); logger.LogInformation("Parsing body from request."); PipelineRequest request = await new BodyReader(httpRequest).GetRequestBodyAsync(); request.Validate(logger); using (var service = PipelineService.GetServiceForRequest(request, logger)) { PipelineDescription result = service.ValidatePipeline(request); logger.LogInformation("ValidatePipeline Function complete."); return new OkObjectResult(JsonConvert.SerializeObject(result)); } } } } ================================================ FILE: Functions/Functions.csproj ================================================ netcoreapp3.1 v3 mrpaulandrew.azure.procfwk mrpaulandrew.azure.procfwk PreserveNewest Always ================================================ FILE: Functions/Helpers/BodyReader.cs ================================================ using System; using System.Collections.Generic; using System.IO; using System.Text; using System.Threading.Tasks; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; using Newtonsoft.Json; namespace mrpaulandrew.azure.procfwk.Helpers { public class BodyReader { public string Body; public BodyReader(HttpRequest httpRequest) { Body = new StreamReader(httpRequest.Body).ReadToEnd(); } public Task GetRequestBody() { PipelineRequest request = JsonConvert.DeserializeObject(Body); return Task.FromResult(request); } public async Task GetRequestBodyAsync() { PipelineRequest request = await GetRequestBody(); return request; } public Task GetRunRequestBody() { PipelineRunRequest request = JsonConvert.DeserializeObject(Body); return Task.FromResult(request); } public async Task GetRunRequestBodyAsync() { PipelineRunRequest request = await GetRunRequestBody(); return request; } } } ================================================ FILE: Functions/Helpers/InvalidRequestException.cs ================================================ using System; using System.Runtime.Serialization; namespace mrpaulandrew.azure.procfwk.Helpers { [Serializable] internal class InvalidRequestException : Exception { public InvalidRequestException() { } public InvalidRequestException(string message) : base(message) { } public InvalidRequestException(string message, Exception innerException) : base(message, innerException) { } protected InvalidRequestException(SerializationInfo info, StreamingContext context) : base(info, context) { } } } ================================================ FILE: Functions/Helpers/KeyVaultClient.cs ================================================ using Azure.Identity; using Azure.Security.KeyVault.Secrets; using System; namespace mrpaulandrew.azure.procfwk.Helpers { internal class KeyVaultClient { private static readonly DefaultAzureCredential defaultCred = new DefaultAzureCredential(); public static string GetSecretFromUri(string secretString) { return GetSecretFromUri(new Uri(secretString)); } public static string GetSecretFromUri(Uri secretUri) { string keyVaultURL = "https://" + secretUri.Host.ToString(); string secretName = secretUri.LocalPath.ToString().Replace("secrets/", "").Replace("/", ""); return CreateKeyVaultClient(keyVaultURL).GetSecret(secretName).Value.Value; } public static string GetSecretFromName(string keyVaultURL, string secretName) { return CreateKeyVaultClient(keyVaultURL).GetSecret(secretName).Value.Value; } private static SecretClient CreateKeyVaultClient(string keyVaultURL) { return new SecretClient(new Uri(keyVaultURL), defaultCred); } } } ================================================ FILE: Functions/Helpers/PipelineRequest.cs ================================================ using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; namespace mrpaulandrew.azure.procfwk.Helpers { public class PipelineRequest { public string TenantId { get; set; } public string ApplicationId { get; set; } public string AuthenticationKey { get; set; } public string SubscriptionId { get; set; } public string ResourceGroupName { get; set; } public string OrchestratorName { get; set; } public string PipelineName { get; set; } public PipelineServiceType? OrchestratorType { get; set; } public Dictionary PipelineParameters; public virtual void Validate(ILogger logger) { // ensure properties not null if ( TenantId == null || ApplicationId == null || AuthenticationKey == null || SubscriptionId == null || ResourceGroupName == null || OrchestratorType == null || OrchestratorName == null || PipelineName == null ) ReportInvalidBody(logger); //other validation if (!CheckGuid(TenantId)) ReportInvalidBody(logger, "Expected Tenant Id to be a GUID."); if (!CheckGuid(SubscriptionId)) ReportInvalidBody(logger, "Expected Subscription Id to be a GUID."); // resolve key vault values if (!CheckGuid(ApplicationId) && CheckUri(ApplicationId)) { logger.LogInformation("Getting applicationId from Key Vault"); ApplicationId = KeyVaultClient.GetSecretFromUri(ApplicationId); } if (CheckUri(AuthenticationKey)) { logger.LogInformation("Getting authenticationKey from Key Vault"); AuthenticationKey = KeyVaultClient.GetSecretFromUri(AuthenticationKey); } } private bool CheckUri(string uriValue) { bool result = Uri.TryCreate(uriValue, UriKind.Absolute, out Uri uriResult) && (uriResult.Scheme == Uri.UriSchemeHttp || uriResult.Scheme == Uri.UriSchemeHttps); return result; } public bool CheckGuid(string idValue) { bool result = Guid.TryParse(idValue, out _); return result; } protected void ReportInvalidBody(ILogger logger) { var msg = "Invalid body."; logger.LogError(msg); throw new InvalidRequestException(msg); } protected void ReportInvalidBody(ILogger logger, string additions) { var msg = "Invalid body. " + additions; logger.LogError(msg); throw new InvalidRequestException(msg); } public Dictionary ParametersAsObjects { get { if (PipelineParameters == null) return null; var dictionary = new Dictionary(); foreach (var key in PipelineParameters.Keys) dictionary.Add(key, PipelineParameters[key]); return dictionary; } } } } ================================================ FILE: Functions/Helpers/PipelineRunRequest.cs ================================================ using System; using Microsoft.Extensions.Logging; namespace mrpaulandrew.azure.procfwk.Helpers { public class PipelineRunRequest : PipelineRequest { public string RunId { get; set; } public bool RecursivePipelineCancel = true; //might provide this as part of the request later public DateTime ActivityQueryStart = DateTime.Now.AddDays(-7); //max duration for eventual RunFilterParameters public DateTime ActivityQueryEnd = DateTime.Now; public override void Validate(ILogger logger) { base.Validate(logger); // ensure properties not null if (RunId == null) ReportInvalidBody(logger); //other validation if (!CheckGuid(RunId)) ReportInvalidBody(logger, "Expected Run Id to be a GUID."); } } } ================================================ FILE: Functions/Helpers/SMTPClient.cs ================================================ using System; using System.Net.Mail; namespace mrpaulandrew.azure.procfwk.Helpers { internal class SMTPClient { public static string FromEmail { get; set; } public static SmtpClient CreateSMTPClient() { string smtpHost = Environment.GetEnvironmentVariable("AppSettingSmtpHost"); int smtpPort = int.Parse(Environment.GetEnvironmentVariable("AppSettingSmtpPort")); string smtpUser = Environment.GetEnvironmentVariable("AppSettingSmtpUser"); string smtpPass = Environment.GetEnvironmentVariable("AppSettingSmtpPass"); FromEmail = Environment.GetEnvironmentVariable("AppSettingFromEmail"); SmtpClient emailClient = new SmtpClient { EnableSsl = true, UseDefaultCredentials = false, //order properties are set is important Credentials = new System.Net.NetworkCredential(smtpUser, smtpPass), DeliveryMethod = SmtpDeliveryMethod.Network, Host = smtpHost, Port = smtpPort }; return emailClient; } } } ================================================ FILE: Functions/Properties/ServiceDependencies/FrameworkSupportFunctions - Zip Deploy/profile.arm.json ================================================ { "$schema": "https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#", "contentVersion": "1.0.0.0", "metadata": { "_dependencyType": "function.windows.consumption" }, "parameters": { "resourceGroupName": { "type": "string", "defaultValue": "ADF.procfwk", "metadata": { "description": "Name of the resource group for the resource. It is recommended to put resources under same resource group for better tracking." } }, "resourceGroupLocation": { "type": "string", "defaultValue": "uksouth", "metadata": { "description": "Location of the resource group. Resource groups could have different location than resources, however by default we use API versions from latest hybrid profile which support all locations for resource types we support." } }, "resourceName": { "type": "string", "defaultValue": "FrameworkSupportFunctions", "metadata": { "description": "Name of the main resource to be created by this template." } }, "resourceLocation": { "type": "string", "defaultValue": "[parameters('resourceGroupLocation')]", "metadata": { "description": "Location of the resource. By default use resource group's location, unless the resource provider is not supported there." } } }, "resources": [ { "type": "Microsoft.Resources/resourceGroups", "name": "[parameters('resourceGroupName')]", "location": "[parameters('resourceGroupLocation')]", "apiVersion": "2019-10-01" }, { "type": "Microsoft.Resources/deployments", "name": "[concat(parameters('resourceGroupName'), 'Deployment', uniqueString(concat(parameters('resourceName'), subscription().subscriptionId)))]", "resourceGroup": "[parameters('resourceGroupName')]", "apiVersion": "2019-10-01", "dependsOn": [ "[parameters('resourceGroupName')]" ], "properties": { "mode": "Incremental", "expressionEvaluationOptions": { "scope": "inner" }, "parameters": { "resourceGroupName": { "value": "[parameters('resourceGroupName')]" }, "resourceGroupLocation": { "value": "[parameters('resourceGroupLocation')]" }, "resourceName": { "value": "[parameters('resourceName')]" }, "resourceLocation": { "value": "[parameters('resourceLocation')]" } }, "template": { "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { "resourceGroupName": { "type": "string" }, "resourceGroupLocation": { "type": "string" }, "resourceName": { "type": "string" }, "resourceLocation": { "type": "string" } }, "variables": { "storage_name": "[toLower(concat('storage', uniqueString(concat(parameters('resourceName'), subscription().subscriptionId))))]", "storage_ResourceId": "[concat('/subscriptions/', subscription().subscriptionId, '/resourceGroups/', parameters('resourceGroupName'), '/providers/Microsoft.Storage/storageAccounts/', variables('storage_name'))]", "function_ResourceId": "[concat('/subscriptions/', subscription().subscriptionId, '/resourceGroups/', parameters('resourceGroupName'), '/providers/Microsoft.Web/sites/', parameters('resourceName'))]" }, "resources": [ { "location": "[parameters('resourceGroupLocation')]", "name": "[variables('storage_name')]", "type": "Microsoft.Storage/storageAccounts", "apiVersion": "2017-10-01", "tags": { "[concat('hidden-related:', concat('/providers/Microsoft.Web/sites/', parameters('resourceName')))]": "empty" }, "properties": { "supportsHttpsTrafficOnly": true }, "sku": { "name": "Standard_LRS" }, "kind": "Storage" }, { "location": "[parameters('resourceLocation')]", "name": "[parameters('resourceName')]", "type": "Microsoft.Web/sites", "apiVersion": "2015-08-01", "dependsOn": [ "[variables('storage_ResourceId')]" ], "kind": "functionapp", "properties": { "name": "[parameters('resourceName')]", "kind": "functionapp", "httpsOnly": true, "reserved": false }, "identity": { "type": "SystemAssigned" }, "resources": [ { "name": "appsettings", "type": "config", "apiVersion": "2015-08-01", "dependsOn": [ "[variables('function_ResourceId')]" ], "properties": { "WEBSITE_CONTENTAZUREFILECONNECTIONSTRING": "[concat('DefaultEndpointsProtocol=https;AccountName=', variables('storage_name'), ';AccountKey=', listKeys(variables('storage_ResourceId'), '2017-10-01').keys[0].value, ';EndpointSuffix=', 'core.windows.net')]", "WEBSITE_CONTENTSHARE": "[toLower(parameters('resourceName'))]", "AzureWebJobsDashboard": "[concat('DefaultEndpointsProtocol=https;AccountName=', variables('storage_name'), ';AccountKey=', listKeys(variables('storage_ResourceId'), '2017-10-01').keys[0].value, ';EndpointSuffix=', 'core.windows.net')]", "AzureWebJobsStorage": "[concat('DefaultEndpointsProtocol=https;AccountName=', variables('storage_name'), ';AccountKey=', listKeys(variables('storage_ResourceId'), '2017-10-01').keys[0].value, ';EndpointSuffix=', 'core.windows.net')]", "FUNCTIONS_EXTENSION_VERSION": "~3", "FUNCTIONS_WORKER_RUNTIME": "dotnet" } } ] } ] } } } ] } ================================================ FILE: Functions/Properties/ServiceDependencies/FrameworkSupportFunctionsBig - Web Deploy/profile.arm.json ================================================ { "$schema": "https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#", "contentVersion": "1.0.0.0", "metadata": { "_dependencyType": "appService.windows" }, "parameters": { "resourceGroupName": { "type": "string", "defaultValue": "ADF.procfwkStressTest", "metadata": { "description": "Name of the resource group for the resource. It is recommended to put resources under same resource group for better tracking." } }, "resourceGroupLocation": { "type": "string", "defaultValue": "uksouth", "metadata": { "description": "Location of the resource group. Resource groups could have different location than resources, however by default we use API versions from latest hybrid profile which support all locations for resource types we support." } }, "resourceName": { "type": "string", "defaultValue": "FrameworkSupportFunctionsBig", "metadata": { "description": "Name of the main resource to be created by this template." } }, "resourceLocation": { "type": "string", "defaultValue": "[parameters('resourceGroupLocation')]", "metadata": { "description": "Location of the resource. By default use resource group's location, unless the resource provider is not supported there." } } }, "variables": { "appServicePlan_name": "[concat('Plan', uniqueString(concat(parameters('resourceName'), subscription().subscriptionId)))]", "appServicePlan_ResourceId": "[concat('/subscriptions/', subscription().subscriptionId, '/resourceGroups/', parameters('resourceGroupName'), '/providers/Microsoft.Web/serverFarms/', variables('appServicePlan_name'))]" }, "resources": [ { "type": "Microsoft.Resources/resourceGroups", "name": "[parameters('resourceGroupName')]", "location": "[parameters('resourceGroupLocation')]", "apiVersion": "2019-10-01" }, { "type": "Microsoft.Resources/deployments", "name": "[concat(parameters('resourceGroupName'), 'Deployment', uniqueString(concat(parameters('resourceName'), subscription().subscriptionId)))]", "resourceGroup": "[parameters('resourceGroupName')]", "apiVersion": "2019-10-01", "dependsOn": [ "[parameters('resourceGroupName')]" ], "properties": { "mode": "Incremental", "template": { "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "resources": [ { "location": "[parameters('resourceLocation')]", "name": "[parameters('resourceName')]", "type": "Microsoft.Web/sites", "apiVersion": "2015-08-01", "tags": { "[concat('hidden-related:', variables('appServicePlan_ResourceId'))]": "empty" }, "dependsOn": [ "[variables('appServicePlan_ResourceId')]" ], "kind": "app", "properties": { "name": "[parameters('resourceName')]", "kind": "app", "httpsOnly": true, "reserved": false, "serverFarmId": "[variables('appServicePlan_ResourceId')]", "siteConfig": { "metadata": [ { "name": "CURRENT_STACK", "value": "dotnetcore" } ] } }, "identity": { "type": "SystemAssigned" } }, { "location": "[parameters('resourceLocation')]", "name": "[variables('appServicePlan_name')]", "type": "Microsoft.Web/serverFarms", "apiVersion": "2015-08-01", "sku": { "name": "S1", "tier": "Standard", "family": "S", "size": "S1" }, "properties": { "name": "[variables('appServicePlan_name')]" } } ] } } } ] } ================================================ FILE: Functions/Services/AzureDataFactoryService.cs ================================================ using System; using System.Threading; using Newtonsoft.Json; using Microsoft.Rest; using Microsoft.Extensions.Logging; using Microsoft.IdentityModel.Clients.ActiveDirectory; using Microsoft.Azure.Management.DataFactory; using Microsoft.Azure.Management.DataFactory.Models; using mrpaulandrew.azure.procfwk.Helpers; namespace mrpaulandrew.azure.procfwk.Services { public class AzureDataFactoryService : PipelineService { private readonly DataFactoryManagementClient _adfManagementClient; private readonly ILogger _logger; public AzureDataFactoryService(PipelineRequest request, ILogger logger) { _logger = logger; _logger.LogInformation("Creating ADF connectivity clients."); //Auth details var context = new AuthenticationContext("https://login.windows.net/" + request.TenantId); var cc = new ClientCredential(request.ApplicationId, request.AuthenticationKey); var result = context.AcquireTokenAsync("https://management.azure.com/", cc).Result; var cred = new TokenCredentials(result.AccessToken); //Management Client _adfManagementClient = new DataFactoryManagementClient(cred) { SubscriptionId = request.SubscriptionId }; } public override PipelineDescription ValidatePipeline(PipelineRequest request) { _logger.LogInformation("Validating ADF pipeline."); try { var pipelineResource = _adfManagementClient.Pipelines.Get ( request.ResourceGroupName, request.OrchestratorName, request.PipelineName ); _logger.LogInformation(pipelineResource.Id.ToString()); return new PipelineDescription() { PipelineExists = "True", PipelineName = pipelineResource.Name, PipelineId = pipelineResource.Id, PipelineType = pipelineResource.Type, ActivityCount = pipelineResource.Activities.Count }; } catch (Microsoft.Rest.Azure.CloudException) //expected exception when pipeline doesnt exist { return new PipelineDescription() { PipelineExists = "False", PipelineName = request.PipelineName, PipelineId = "Unknown", PipelineType = "Unknown", ActivityCount = 0 }; } catch (Exception ex) //other unknown issue { _logger.LogInformation(ex.Message); _logger.LogInformation(ex.GetType().ToString()); throw new InvalidRequestException("Failed to validate pipeline. ", ex); } } public override PipelineRunStatus ExecutePipeline(PipelineRequest request) { if (request.PipelineParameters == null) _logger.LogInformation("Calling pipeline without parameters."); else _logger.LogInformation("Calling pipeline with parameters."); var runResponse = _adfManagementClient.Pipelines.CreateRunWithHttpMessagesAsync ( request.ResourceGroupName, request.OrchestratorName, request.PipelineName, parameters: request.ParametersAsObjects ).Result.Body; _logger.LogInformation("Pipeline run ID: " + runResponse.RunId); //Wait and check for pipeline to start... PipelineRun pipelineRun; _logger.LogInformation("Checking ADF pipeline status."); while (true) { pipelineRun = _adfManagementClient.PipelineRuns.Get ( request.ResourceGroupName, request.OrchestratorName, runResponse.RunId ); _logger.LogInformation("Waiting for pipeline to start, current status: " + pipelineRun.Status); if (pipelineRun.Status != "Queued") break; Thread.Sleep(internalWaitDuration); } return new PipelineRunStatus() { PipelineName = request.PipelineName, RunId = runResponse.RunId, ActualStatus = pipelineRun.Status }; } public override PipelineRunStatus CancelPipeline(PipelineRunRequest request) { _logger.LogInformation("Getting ADF pipeline current status."); PipelineRun pipelineRun; pipelineRun = _adfManagementClient.PipelineRuns.Get ( request.ResourceGroupName, request.OrchestratorName, request.RunId ); //Defensive check PipelineNameCheck(request.PipelineName, pipelineRun.PipelineName); if (pipelineRun.Status == "InProgress" || pipelineRun.Status == "Queued") { _logger.LogInformation("Attempting to cancel ADF pipeline."); _adfManagementClient.PipelineRuns.Cancel ( request.ResourceGroupName, request.OrchestratorName, request.RunId, isRecursive : request.RecursivePipelineCancel ); } else { _logger.LogInformation("ADF pipeline status: " + pipelineRun.Status); throw new InvalidRequestException("Target pipeline is not in a state that can be cancelled."); } //wait for cancelled state _logger.LogInformation("Checking ADF pipeline status after cancel request."); while (true) { pipelineRun = _adfManagementClient.PipelineRuns.Get ( request.ResourceGroupName, request.OrchestratorName, request.RunId ); _logger.LogInformation("Waiting for pipeline to cancel, current status: " + pipelineRun.Status); if (pipelineRun.Status == "Cancelled") break; Thread.Sleep(internalWaitDuration); } //Final return detail return new PipelineRunStatus() { PipelineName = request.PipelineName, RunId = request.RunId, ActualStatus = pipelineRun.Status.Replace("Canceling", "Cancelling") //microsoft typo }; } public override PipelineRunStatus GetPipelineRunStatus(PipelineRunRequest request) { _logger.LogInformation("Checking ADF pipeline status."); //Get pipeline status with provided run id PipelineRun pipelineRun; pipelineRun = _adfManagementClient.PipelineRuns.Get ( request.ResourceGroupName, request.OrchestratorName, request.RunId ); //Defensive check PipelineNameCheck(request.PipelineName, pipelineRun.PipelineName); _logger.LogInformation("ADF pipeline status: " + pipelineRun.Status); //Defensive check PipelineNameCheck(request.PipelineName, pipelineRun.PipelineName); //Final return detail return new PipelineRunStatus() { PipelineName = request.PipelineName, RunId = pipelineRun.RunId, ActualStatus = pipelineRun.Status.Replace("Canceling", "Cancelling") //microsoft typo }; } public override PipelineErrorDetail GetPipelineRunActivityErrors(PipelineRunRequest request) { PipelineRun pipelineRun = _adfManagementClient.PipelineRuns.Get ( request.ResourceGroupName, request.OrchestratorName, request.RunId ); //Defensive check PipelineNameCheck(request.PipelineName, pipelineRun.PipelineName); _logger.LogInformation("Create pipeline Activity Runs query filters."); RunFilterParameters filterParams = new RunFilterParameters ( request.ActivityQueryStart, request.ActivityQueryEnd ); _logger.LogInformation("Querying ADF pipeline for Activity Runs."); ActivityRunsQueryResponse queryResponse = _adfManagementClient.ActivityRuns.QueryByPipelineRun ( request.ResourceGroupName, request.OrchestratorName, request.RunId, filterParams ); //Create initial output content PipelineErrorDetail output = new PipelineErrorDetail() { PipelineName = request.PipelineName, ActualStatus = pipelineRun.Status, RunId = request.RunId, ResponseCount = queryResponse.Value.Count }; _logger.LogInformation("Pipeline status: " + pipelineRun.Status); _logger.LogInformation("Activities found in pipeline response: " + queryResponse.Value.Count.ToString()); //Loop over activities in pipeline run foreach (ActivityRun activity in queryResponse.Value) { if (activity.Error == null) { continue; //only want errors } //Parse error output to customise output dynamic outputBlockInner = JsonConvert.DeserializeObject(activity.Error.ToString()); string errorCode = outputBlockInner?.errorCode; string errorType = outputBlockInner?.failureType; string errorMessage = outputBlockInner?.message; _logger.LogInformation("Activity run id: " + activity.ActivityRunId); _logger.LogInformation("Activity name: " + activity.ActivityName); _logger.LogInformation("Activity type: " + activity.ActivityType); _logger.LogInformation("Error message: " + errorMessage); output.Errors.Add(new FailedActivity() { ActivityRunId = activity.ActivityRunId, ActivityName = activity.ActivityName, ActivityType = activity.ActivityType, ErrorCode = errorCode, ErrorType = errorType, ErrorMessage = errorMessage }); } return output; } public override void Dispose() { _adfManagementClient?.Dispose(); } } } ================================================ FILE: Functions/Services/AzureSynapseService.cs ================================================ using System; using System.Threading; using System.Collections.Generic; using Newtonsoft.Json; using Microsoft.Rest; using Microsoft.Extensions.Logging; using Microsoft.IdentityModel.Clients.ActiveDirectory; using Microsoft.Azure.Management.Synapse; using Azure.Core; using Azure.Identity; using Azure.Analytics.Synapse.Artifacts; using Azure.Analytics.Synapse.Artifacts.Models; using mrpaulandrew.azure.procfwk.Helpers; namespace mrpaulandrew.azure.procfwk.Services { public class AzureSynapseService : PipelineService { private readonly SynapseManagementClient _synManagementClient; private readonly PipelineClient _pipelineClient; private readonly PipelineRunClient _pipelineRunClient; private readonly ILogger _logger; public AzureSynapseService(PipelineRequest request, ILogger logger) { _logger = logger; _logger.LogInformation("Creating SYN connectivity clients."); //Auth details var context = new AuthenticationContext("https://login.windows.net/" + request.TenantId); var cc = new ClientCredential(request.ApplicationId, request.AuthenticationKey); var result = context.AcquireTokenAsync("https://management.azure.com/", cc).Result; var cred = new TokenCredentials(result.AccessToken); //Management Client _synManagementClient = new SynapseManagementClient(cred) { SubscriptionId = request.SubscriptionId }; //Pipeline Clients Uri synapseDevEndpoint = new Uri("https://" + request.OrchestratorName.ToLower() + ".dev.azuresynapse.net"); TokenCredential token = new ClientSecretCredential ( request.TenantId, request.ApplicationId, request.AuthenticationKey ); _pipelineClient = new PipelineClient(synapseDevEndpoint, token); _pipelineRunClient = new PipelineRunClient(synapseDevEndpoint, token); } public override PipelineDescription ValidatePipeline(PipelineRequest request) { _logger.LogInformation("Validating SYN pipeline."); PipelineResource pipelineResource; try { pipelineResource = _pipelineClient.GetPipeline ( request.PipelineName ); _logger.LogInformation(pipelineResource.Id.ToString()); return new PipelineDescription() { PipelineExists = "True", PipelineName = pipelineResource.Name, PipelineId = pipelineResource.Id, PipelineType = pipelineResource.Type, ActivityCount = pipelineResource.Activities.Count }; } catch (System.InvalidOperationException) //for bug in underlying activity classes, pipeline does exist { return new PipelineDescription() { PipelineExists = "True", PipelineName = request.PipelineName, PipelineId = "Unknown", PipelineType = "Unknown", ActivityCount = 0 }; } catch (Azure.RequestFailedException) //expected exception when pipeline doesnt exist { return new PipelineDescription() { PipelineExists = "False", PipelineName = request.PipelineName, PipelineId = "Unknown", PipelineType = "Unknown", ActivityCount = 0 }; } catch (Exception ex) //other unknown issue { _logger.LogInformation(ex.Message); _logger.LogInformation(ex.GetType().ToString()); throw new InvalidRequestException("Failed to validate pipeline. ", ex); } } public override PipelineRunStatus ExecutePipeline(PipelineRequest request) { if (request.PipelineParameters == null) _logger.LogInformation("Calling pipeline without parameters."); else _logger.LogInformation("Calling pipeline with parameters."); CreateRunResponse runResponse; runResponse = _pipelineClient.CreatePipelineRun ( request.PipelineName, parameters: request.ParametersAsObjects ); _logger.LogInformation("Pipeline run ID: " + runResponse.RunId); //Wait and check for pipeline to start... PipelineRun pipelineRun; _logger.LogInformation("Checking ADF pipeline status."); while (true) { pipelineRun = _pipelineRunClient.GetPipelineRun ( runResponse.RunId ); _logger.LogInformation("Waiting for pipeline to start, current status: " + pipelineRun.Status); if (pipelineRun.Status != "Queued") break; Thread.Sleep(internalWaitDuration); } return new PipelineRunStatus() { PipelineName = request.PipelineName, RunId = runResponse.RunId, ActualStatus = pipelineRun.Status }; } public override PipelineRunStatus CancelPipeline(PipelineRunRequest request) { _logger.LogInformation("Getting SYN pipeline current status."); PipelineRun pipelineRun; pipelineRun = _pipelineRunClient.GetPipelineRun ( request.RunId ); //Defensive check PipelineNameCheck(request.PipelineName, pipelineRun.PipelineName); if (pipelineRun.Status == "InProgress" || pipelineRun.Status == "Queued") { _logger.LogInformation("Attempting to cancel SYN pipeline."); _pipelineRunClient.CancelPipelineRun ( request.RunId, isRecursive: request.RecursivePipelineCancel ); } else { _logger.LogInformation("ADF pipeline status: " + pipelineRun.Status); throw new InvalidRequestException("Target pipeline is not in a state that can be cancelled."); } //wait for cancelled state _logger.LogInformation("Checking ADF pipeline status after cancel request."); while (true) { pipelineRun = _pipelineRunClient.GetPipelineRun ( request.RunId ); _logger.LogInformation("Waiting for pipeline to cancel, current status: " + pipelineRun.Status); if (pipelineRun.Status == "Cancelled") break; Thread.Sleep(internalWaitDuration); } //Final return detail return new PipelineRunStatus() { PipelineName = request.PipelineName, RunId = request.RunId, ActualStatus = pipelineRun.Status }; } public override PipelineRunStatus GetPipelineRunStatus(PipelineRunRequest request) { _logger.LogInformation("Getting SYN pipeline status."); //Get pipeline status with provided run id PipelineRun pipelineRun; pipelineRun = _pipelineRunClient.GetPipelineRun ( request.RunId ); //Defensive check PipelineNameCheck(request.PipelineName, pipelineRun.PipelineName); _logger.LogInformation("SYN pipeline status: " + pipelineRun.Status); //Final return detail return new PipelineRunStatus() { PipelineName = request.PipelineName, RunId = pipelineRun.RunId, ActualStatus = pipelineRun.Status.Replace("Canceling", "Cancelling") //microsoft typo }; } public override PipelineErrorDetail GetPipelineRunActivityErrors(PipelineRunRequest request) { PipelineRun pipelineRun = _pipelineRunClient.GetPipelineRun ( request.RunId ); //Defensive check PipelineNameCheck(request.PipelineName, pipelineRun.PipelineName); _logger.LogInformation("Create pipeline Activity Runs query filters."); RunFilterParameters filterParams = new RunFilterParameters ( request.ActivityQueryStart, request.ActivityQueryEnd ); _logger.LogInformation("Querying SYN pipeline for Activity Runs."); ActivityRunsQueryResponse queryResponse = _pipelineRunClient.QueryActivityRuns ( request.PipelineName, request.RunId, filterParams ); //Create initial output content PipelineErrorDetail output = new PipelineErrorDetail() { PipelineName = request.PipelineName, ActualStatus = pipelineRun.Status, RunId = request.RunId, ResponseCount = queryResponse.Value.Count }; _logger.LogInformation("Pipeline status: " + pipelineRun.Status); _logger.LogInformation("Activities found in pipeline response: " + queryResponse.Value.Count.ToString()); //Loop over activities in pipeline run foreach (ActivityRun activity in queryResponse.Value) { if (activity.Error == null) { continue; //only want errors } //Parse error output to customise output var json = JsonConvert.SerializeObject(activity.Error); Dictionary errorContent = JsonConvert.DeserializeObject>(json); _logger.LogInformation("Activity run id: " + activity.ActivityRunId); _logger.LogInformation("Activity name: " + activity.ActivityName); _logger.LogInformation("Activity type: " + activity.ActivityType); _logger.LogInformation("Error message: " + errorContent["message"].ToString()); output.Errors.Add(new FailedActivity() { ActivityRunId = activity.ActivityRunId, ActivityName = activity.ActivityName, ActivityType = activity.ActivityType, ErrorCode = errorContent["errorCode"].ToString(), ErrorType = errorContent["failureType"].ToString(), ErrorMessage = errorContent["message"].ToString() }); } return output; } public override void Dispose() { _synManagementClient?.Dispose(); //_pipelineClient?.Dispose(); not yet supported //_pipelineRunClient?.Dispose(); not yet supported } } } ================================================ FILE: Functions/Services/PipelineService.cs ================================================ using mrpaulandrew.azure.procfwk.Helpers; using Microsoft.Extensions.Logging; using System; namespace mrpaulandrew.azure.procfwk.Services { public abstract class PipelineService : IDisposable { public const int internalWaitDuration = 5000; //ms public static PipelineService GetServiceForRequest(PipelineRequest pr, ILogger logger) { if (pr.OrchestratorType == PipelineServiceType.ADF) return new AzureDataFactoryService(pr, logger); if (pr.OrchestratorType == PipelineServiceType.SYN) return new AzureSynapseService(pr, logger); throw new InvalidRequestException ("Unsupported orchestrator type: " + (pr.OrchestratorType?.ToString() ?? "")); } public abstract PipelineDescription ValidatePipeline(PipelineRequest request); public abstract PipelineRunStatus ExecutePipeline(PipelineRequest request); public abstract PipelineRunStatus CancelPipeline(PipelineRunRequest request); public abstract PipelineRunStatus GetPipelineRunStatus(PipelineRunRequest request); public abstract PipelineErrorDetail GetPipelineRunActivityErrors(PipelineRunRequest request); protected void PipelineNameCheck(string requestName, string foundName) { if (requestName.ToUpper() != foundName.ToUpper()) { throw new InvalidRequestException($"Pipeline name mismatch. Provided pipeline name does not match the provided Run Id. Expected name: {foundName}"); } } public abstract void Dispose(); } } ================================================ FILE: Functions/Services/PipelineServiceType.cs ================================================ namespace mrpaulandrew.azure.procfwk { public enum PipelineServiceType { ADF, SYN } } ================================================ FILE: Functions/Services/Returns/PipelineDescription.cs ================================================ namespace mrpaulandrew.azure.procfwk.Services { public class PipelineDescription { public string PipelineExists { get; set; } public string PipelineName { get; set; } public string PipelineId { get; set; } public string PipelineType { get; set; } public int ActivityCount { get; set; } } } ================================================ FILE: Functions/Services/Returns/PipelineErrorDetail.cs ================================================ using System.Collections.Generic; namespace mrpaulandrew.azure.procfwk.Services { public class PipelineErrorDetail : PipelineRunStatus { public PipelineErrorDetail() { Errors = new List(); } public int ResponseCount { get; set; } public int ResponseErrorCount { get { return Errors.Count; } } public List Errors { get; } } public class FailedActivity { public string ActivityRunId { get; internal set; } public string ActivityName { get; internal set; } public string ActivityType { get; internal set; } public string ErrorCode { get; internal set; } public string ErrorType { get; internal set; } public string ErrorMessage { get; internal set; } } } ================================================ FILE: Functions/Services/Returns/PipelineRunStatus.cs ================================================ using System.Collections.Generic; namespace mrpaulandrew.azure.procfwk.Services { public class PipelineRunStatus { private const string Running = "Running"; private const string Complete = "Complete"; public string PipelineName { get; set; } public string RunId { get; set; } public string ActualStatus { get; set; } public string SimpleStatus { get { return ConvertPipelineStatus(ActualStatus); } } private string ConvertPipelineStatus(string actualStatus) { string simpleStatus = actualStatus switch { "Queued" => Running, "InProgress" => Running, "Canceling" => Running, //microsoft typo "Cancelling" => Running, _ => Complete, }; return simpleStatus; } } } ================================================ FILE: Functions/host.json ================================================ { "version": "2.0" } ================================================ FILE: Functions/template_local.settings.json ================================================ { "IsEncrypted": false, "Values": { "AzureWebJobsStorage": "UseDevelopmentStorage=true", "FUNCTIONS_WORKER_RUNTIME": "dotnet", "AppSettingSmtpHost": "my.smtpservice.com", "AppSettingSmtpPort": 1234, "AppSettingSmtpUser": "AlertMailboxUser", "AppSettingSmtpPass": "Passw0rd123!", "AppSettingFromEmail": "noreply@adfprocfwk.com" } } ================================================ FILE: LICENSE ================================================ MIT License Copyright (c) 2020 Paul Andrew (mrpaulandrew.com) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. THE SOFTWARE IS ALSO IN NO WAY ASOCIATED WITH OR SUPPORTED BY: - Altius Consulting Limited - Cloudpoint Limited - Farah Bidco Limited - Farah Midco Limited - Farah Topco Limited - Avanade UK Ltd - Avanade Europe Services Ltd - Avanade Europe Holdings Ltd - Avanade International Corporation ================================================ FILE: MetadataDB/MetadataDB.refactorlog ================================================  ================================================ FILE: MetadataDB/MetadataDB.sqlproj ================================================  Debug AnyCPU MetadataDB 2.0 4.1 {202ebf84-a56b-4999-92a3-10f7ffe4ef25} Microsoft.Data.Tools.Schema.Sql.SqlAzureV12DatabaseSchemaProvider Database MetadataDB MetadataDB 1033,CI BySchemaAndSchemaType True v4.7.2 CS Properties False True True SQL_Latin1_General_CP1_CI_AS PRIMARY procfwk bin\Release\ $(MSBuildProjectName).sql False pdbonly true false true prompt 4 bin\Debug\ $(MSBuildProjectName).sql false true full false true true prompt 4 11.0 True 11.0 $(DacPacRootPath)\Extensions\Microsoft\SQLDB\Extensions\SqlServer\AzureV12\SqlSchemas\master.dacpac False master ================================================ FILE: MetadataDB/Scripts/Alter Database Scale.sql ================================================ ALTER DATABASE [FrameworkMetadataDev] MODIFY (EDITION ='Standard', SERVICE_OBJECTIVE = 'S2', MAXSIZE = 50 GB); ================================================ FILE: MetadataDB/Scripts/Handy Selects.sql ================================================ SELECT * FROM [procfwk].[BatchExecution] SELECT * FROM [procfwk].[CurrentExecution] SELECT * FROM [procfwkReporting].[CurrentExecutionSummary] SELECT * FROM [procfwk].[CurrentProperties] SELECT * FROM [procfwk].[Batches] SELECT * FROM [procfwk].[BatchStageLink] SELECT * FROM [procfwk].[Orchestrators] SELECT * FROM [procfwk].[Stages] SELECT * FROM [procfwk].[Pipelines] --SELECT * FROM [procfwk].[PipelineParameters] SELECT * FROM [procfwk].[ExecutionLog] SELECT * FROM [procfwk].[ErrorLog] --SELECT * FROM [dbo].[ServicePrincipals] ================================================ FILE: MetadataDB/Scripts/LogData/ErrorLogBackup.sql ================================================ IF OBJECT_ID(N'[dbo].[ErrorLogBackup]') IS NOT NULL DROP TABLE [dbo].[ErrorLogBackup]; IF OBJECT_ID(N'[procfwk].[ErrorLog]') IS NOT NULL --check for new deployments BEGIN SELECT * INTO [dbo].[ErrorLogBackup] FROM [procfwk].[ErrorLog]; END; ================================================ FILE: MetadataDB/Scripts/LogData/ErrorLogRestore.sql ================================================ DECLARE @ErrColumns VARCHAR(MAX) = ''; DECLARE @ErrValues VARCHAR(MAX) = ''; DECLARE @ErrSQL VARCHAR(MAX) = ''; IF EXISTS ( SELECT * FROM sys.objects o INNER JOIN sys.schemas s ON o.[schema_id] = s.[schema_id] WHERE o.[name] = 'ErrorLog' AND s.[name] = 'procfwk' ) AND EXISTS ( SELECT * FROM sys.objects o INNER JOIN sys.schemas s ON o.[schema_id] = s.[schema_id] WHERE o.[name] = 'ErrorLogBackup' AND s.[name] = 'dbo' ) BEGIN ;WITH oldTableColumns AS ( SELECT c.[name] AS 'ColName' FROM sys.objects o INNER JOIN sys.schemas s ON o.[schema_id] = s.[schema_id] INNER JOIN sys.columns c ON o.[object_id] = c.[object_id] WHERE s.[name] = 'dbo' AND o.[name] = 'ErrorLogBackup' AND c.[name] != 'LogId' ), newTableColumns AS ( SELECT c.[column_id] AS 'ColId', c.[name] AS 'ColName' FROM sys.objects o INNER JOIN sys.schemas s ON o.[schema_id] = s.[schema_id] INNER JOIN sys.columns c ON o.[object_id] = c.[object_id] WHERE s.[name] = 'procfwk' AND o.[name] = 'ErrorLog' AND c.[name] != 'LogId' ) SELECT @ErrColumns += QUOTENAME(newTableColumns.[ColName]) + ',' + CHAR(13), @ErrValues += ISNULL(QUOTENAME(oldTableColumns.[ColName]),'NULL AS ''' + newTableColumns.[ColName] + '''' ) + ',' + CHAR(13) FROM newTableColumns LEFT OUTER JOIN oldTableColumns ON newTableColumns.[ColName] = oldTableColumns.[ColName]; SET @ErrColumns = LEFT(@ErrColumns,LEN(@ErrColumns)-2); SET @ErrValues = LEFT(@ErrValues,LEN(@ErrValues)-2); SET @ErrSQL = ' INSERT INTO [procfwk].[ErrorLog] ( ' + @ErrColumns + ' ) SELECT ' + @ErrValues + ' FROM [dbo].[ErrorLogBackup] '; PRINT @ErrSQL; EXEC(@ErrSQL); DECLARE @ErrBefore INT = (SELECT COUNT(0) FROM [dbo].[ErrorLogBackup]); DECLARE @ErrAfter INT = (SELECT COUNT(0) FROM [procfwk].[ErrorLog]); IF (@ErrBefore = @ErrAfter) BEGIN DROP TABLE [dbo].[ErrorLogBackup] END; END ================================================ FILE: MetadataDB/Scripts/LogData/ExecutionLogBackup.sql ================================================ IF OBJECT_ID(N'[dbo].[ExecutionLogBackup]') IS NOT NULL DROP TABLE [dbo].[ExecutionLogBackup]; IF OBJECT_ID(N'[procfwk].[ExecutionLog]') IS NOT NULL --check for new deployments BEGIN SELECT * INTO [dbo].[ExecutionLogBackup] FROM [procfwk].[ExecutionLog]; END; ================================================ FILE: MetadataDB/Scripts/LogData/ExecutionLogRestore.sql ================================================ DECLARE @Columns VARCHAR(MAX) = ''; DECLARE @Values VARCHAR(MAX) = ''; DECLARE @SQL VARCHAR(MAX) = ''; IF EXISTS ( SELECT * FROM sys.objects o INNER JOIN sys.schemas s ON o.[schema_id] = s.[schema_id] WHERE o.[name] = 'ExecutionLog' AND s.[name] = 'procfwk' ) AND EXISTS ( SELECT * FROM sys.objects o INNER JOIN sys.schemas s ON o.[schema_id] = s.[schema_id] WHERE o.[name] = 'ExecutionLogBackup' AND s.[name] = 'dbo' ) BEGIN ;WITH oldTableColumns AS ( SELECT c.[name] AS ColName FROM sys.objects o INNER JOIN sys.schemas s ON o.[schema_id] = s.[schema_id] INNER JOIN sys.columns c ON o.[object_id] = c.[object_id] WHERE s.[name] = 'dbo' AND o.[name] = 'ExecutionLogBackup' AND c.[name] <> 'LogId' ), newTableColumns AS ( SELECT c.[column_id] AS ColId, c.[name] AS ColName FROM sys.objects o INNER JOIN sys.schemas s ON o.[schema_id] = s.[schema_id] INNER JOIN sys.columns c ON o.[object_id] = c.[object_id] WHERE s.[name] = 'procfwk' AND o.[name] = 'ExecutionLog' AND c.[name] <> 'LogId' ) SELECT @Columns += QUOTENAME(newTableColumns.[ColName]) + ',' + CHAR(13), @Values += ISNULL(QUOTENAME(oldTableColumns.[ColName]),'NULL AS ''' + newTableColumns.[ColName] + '''' ) + ',' + CHAR(13) FROM newTableColumns LEFT OUTER JOIN oldTableColumns ON newTableColumns.[ColName] = oldTableColumns.[ColName]; SET @Columns = LEFT(@Columns,LEN(@Columns)-2); SET @Values = LEFT(@Values,LEN(@Values)-2); SET @SQL = ' INSERT INTO [procfwk].[ExecutionLog] ( ' + @Columns + ' ) SELECT ' + @Values + ' FROM [dbo].[ExecutionLogBackup] '; PRINT @SQL; EXEC(@SQL); DECLARE @Before INT = (SELECT COUNT(0) FROM [dbo].[ExecutionLogBackup]); DECLARE @After INT = (SELECT COUNT(0) FROM [procfwk].[ExecutionLog]); IF (@Before = @After) BEGIN DROP TABLE [dbo].[ExecutionLogBackup] END; END; ================================================ FILE: MetadataDB/Scripts/Metadata/AlertOutcomes.sql ================================================ EXEC [procfwkHelpers].[SetDefaultAlertOutcomes]; ================================================ FILE: MetadataDB/Scripts/Metadata/DeleteAll.sql ================================================  --BatchExecution IF OBJECT_ID(N'[procfwk].[BatchExecution]') IS NOT NULL BEGIN TRUNCATE TABLE [procfwk].[BatchExecution]; END; --CurrentExecution IF OBJECT_ID(N'[procfwk].[CurrentExecution]') IS NOT NULL BEGIN TRUNCATE TABLE [procfwk].[CurrentExecution]; END; --ExecutionLog IF OBJECT_ID(N'[procfwk].[ExecutionLog]') IS NOT NULL BEGIN TRUNCATE TABLE [procfwk].[ExecutionLog]; END --ErrorLog IF OBJECT_ID(N'[procfwk].[ExecutionLog]') IS NOT NULL BEGIN TRUNCATE TABLE [procfwk].[ErrorLog]; END --BatchStageLink IF OBJECT_ID(N'[procfwk].[BatchStageLink]') IS NOT NULL BEGIN DELETE FROM [procfwk].[BatchStageLink]; END; --Batches IF OBJECT_ID(N'[procfwk].[Batches]') IS NOT NULL BEGIN DELETE FROM [procfwk].[Batches]; END; --PipelineDependencies IF OBJECT_ID(N'[procfwk].[PipelineDependencies]') IS NOT NULL BEGIN DELETE FROM [procfwk].[PipelineDependencies]; DBCC CHECKIDENT ('[procfwk].[PipelineDependencies]', RESEED, 0); END; --PipelineAlertLink IF OBJECT_ID(N'[procfwk].[PipelineAlertLink]') IS NOT NULL BEGIN DELETE FROM [procfwk].[PipelineAlertLink]; DBCC CHECKIDENT ('[procfwk].[PipelineAlertLink]', RESEED, 0); END; --Recipients IF OBJECT_ID(N'[procfwk].[Recipients]') IS NOT NULL BEGIN DELETE FROM [procfwk].[Recipients]; DBCC CHECKIDENT ('[procfwk].[Recipients]', RESEED, 0); END; --AlertOutcomes IF OBJECT_ID(N'[procfwk].[AlertOutcomes]') IS NOT NULL BEGIN TRUNCATE TABLE [procfwk].[AlertOutcomes]; END; --PipelineAuthLink IF OBJECT_ID(N'[procfwk].[PipelineAuthLink]') IS NOT NULL BEGIN DELETE FROM [procfwk].[PipelineAuthLink]; DBCC CHECKIDENT ('[procfwk].[PipelineAuthLink]', RESEED, 0); END; --ServicePrincipals IF OBJECT_ID(N'[dbo].[ServicePrincipals]') IS NOT NULL BEGIN DELETE FROM [dbo].[ServicePrincipals]; DBCC CHECKIDENT ('[dbo].[ServicePrincipals]', RESEED, 0); END; --Properties IF OBJECT_ID(N'[procfwk].[Properties]') IS NOT NULL BEGIN DELETE FROM [procfwk].[Properties]; DBCC CHECKIDENT ('[procfwk].[Properties]', RESEED, 0); END; --PipelineParameters IF OBJECT_ID(N'[procfwk].[PipelineParameters]') IS NOT NULL BEGIN DELETE FROM [procfwk].[PipelineParameters]; DBCC CHECKIDENT ('[procfwk].[PipelineParameters]', RESEED, 0); END; --Pipelines IF OBJECT_ID(N'[procfwk].[Pipelines]') IS NOT NULL BEGIN DELETE FROM [procfwk].[Pipelines]; DBCC CHECKIDENT ('[procfwk].[Pipelines]', RESEED, 0); END; --Orchestrators IF EXISTS ( SELECT * FROM sys.objects o INNER JOIN sys.schemas s ON o.[schema_id] = s.[schema_id] WHERE o.[name] = 'DataFactorys' AND s.[name] = 'procfwk' AND o.[type] = 'U' --Check for tables as created synonyms to support backwards compatability ) BEGIN DELETE FROM [procfwk].[DataFactorys]; END; IF OBJECT_ID(N'[procfwk].[Orchestrators]') IS NOT NULL BEGIN DELETE FROM [procfwk].[Orchestrators]; DBCC CHECKIDENT ('[procfwk].[Orchestrators]', RESEED, 0); END; --Stages IF OBJECT_ID(N'[procfwk].[Stages]') IS NOT NULL BEGIN DELETE FROM [procfwk].[Stages]; DBCC CHECKIDENT ('[procfwk].[Stages]', RESEED, 0); END; --Subscriptions IF OBJECT_ID(N'[procfwk].[Subscriptions]') IS NOT NULL BEGIN DELETE FROM [procfwk].[Subscriptions]; END; --Tenants IF OBJECT_ID(N'[procfwk].[Tenants]') IS NOT NULL BEGIN DELETE FROM [procfwk].[Tenants]; END; ================================================ FILE: MetadataDB/Scripts/Metadata/DropLegacyObjects.sql ================================================ IF OBJECT_ID(N'tempdb..#DropLegacyObjects') IS NOT NULL DROP PROCEDURE #DropLegacyObjects; GO CREATE PROCEDURE #DropLegacyObjects ( @ObjectName NVARCHAR(128), @ObjectType CHAR(2) ) AS BEGIN DECLARE @DDLType NVARCHAR(128) IF EXISTS ( SELECT * FROM sys.objects o INNER JOIN sys.schemas s ON o.[schema_id] = s.[schema_id] WHERE o.[Name] = @ObjectName AND s.[name] = 'procfwk' AND o.[type] = @ObjectType ) BEGIN SELECT @DDLType = CASE @ObjectType WHEN 'P' THEN 'PROCEDURE' WHEN 'V' THEN 'VIEW' WHEN 'FN' THEN 'FUNCTION' END; EXEC('DROP ' + @DDLType + ' [procfwk].[' + @ObjectName + '];') END; END; GO EXEC #DropLegacyObjects 'AddProperty', 'P'; EXEC #DropLegacyObjects 'GetExecutionDetails', 'P'; EXEC #DropLegacyObjects 'AddRecipientPipelineAlerts', 'P'; EXEC #DropLegacyObjects 'DeleteRecipientAlerts', 'P'; EXEC #DropLegacyObjects 'CheckStageAndPiplineIntegrity', 'P'; EXEC #DropLegacyObjects 'AddPipelineDependant', 'P'; EXEC #DropLegacyObjects 'AddServicePrincipalWrapper', 'P'; EXEC #DropLegacyObjects 'AddServicePrincipalUrls', 'P'; EXEC #DropLegacyObjects 'AddServicePrincipal', 'P'; EXEC #DropLegacyObjects 'DeleteServicePrincipal', 'P'; EXEC #DropLegacyObjects 'CheckForValidURL', 'FN'; EXEC #DropLegacyObjects 'PipelineDependencyChains', 'V'; EXEC #DropLegacyObjects 'AverageStageDuration', 'V'; EXEC #DropLegacyObjects 'CompleteExecutionErrorLog', 'V'; EXEC #DropLegacyObjects 'CompleteExecutionLog', 'V'; EXEC #DropLegacyObjects 'CurrentExecutionSummary', 'V'; EXEC #DropLegacyObjects 'LastExecution', 'V'; EXEC #DropLegacyObjects 'LastExecutionSummary', 'V'; EXEC #DropLegacyObjects 'WorkerParallelismOverTime', 'V'; --replaced with new precursor concept in v1.8.5: IF OBJECT_ID(N'[dbo].[SetRandomWaitValues]') IS NOT NULL DROP PROCEDURE [dbo].[SetRandomWaitValues]; ================================================ FILE: MetadataDB/Scripts/Metadata/DropLegacyTables.sql ================================================ --PipelineProcesses IF EXISTS ( SELECT * FROM sys.objects o INNER JOIN sys.schemas s ON o.[schema_id] = s.[schema_id] WHERE o.[name] = 'PipelineProcesses' AND s.[name] = 'procfwk' AND o.[type] = 'U' --Check for tables as created synonyms to support backwards compatability ) BEGIN --drop just to avoid constraints IF OBJECT_ID(N'[procfwk].[PipelineParameters]') IS NOT NULL DROP TABLE [procfwk].[PipelineParameters]; IF OBJECT_ID(N'[procfwk].[PipelineAuthLink]') IS NOT NULL DROP TABLE [procfwk].[PipelineAuthLink]; SELECT * INTO [dbo].[zz_PipelineProcesses] FROM [procfwk].[PipelineProcesses]; DROP TABLE [procfwk].[PipelineProcesses]; END --ProcessingStageDetails IF EXISTS ( SELECT * FROM sys.objects o INNER JOIN sys.schemas s ON o.[schema_id] = s.[schema_id] WHERE o.[name] = 'ProcessingStageDetails' AND s.[name] = 'procfwk' AND o.[type] = 'U' --Check for tables as created synonyms to support backwards compatability ) BEGIN SELECT * INTO [dbo].[zz_ProcessingStageDetails] FROM [procfwk].[ProcessingStageDetails]; DROP TABLE [procfwk].[ProcessingStageDetails]; END; --DataFactoryDetails IF EXISTS ( SELECT * FROM sys.objects o INNER JOIN sys.schemas s ON o.[schema_id] = s.[schema_id] WHERE o.[name] = 'DataFactoryDetails' AND s.[name] = 'procfwk' AND o.[type] = 'U' --Check for tables as created synonyms to support backwards compatability ) BEGIN SELECT * INTO [dbo].[zz_DataFactoryDetails] FROM [procfwk].[DataFactoryDetails]; DROP TABLE [procfwk].[DataFactoryDetails]; END; --DataFactorys IF EXISTS ( SELECT * FROM sys.objects o INNER JOIN sys.schemas s ON o.[schema_id] = s.[schema_id] WHERE o.[name] = 'DataFactorys' AND s.[name] = 'procfwk' AND o.[type] = 'U' --Check for tables as created synonyms to support backwards compatability ) BEGIN SELECT * INTO [dbo].[zz_DataFactorys] FROM [procfwk].[DataFactorys]; END; ================================================ FILE: MetadataDB/Scripts/Metadata/Orchestrators.sql ================================================ EXEC [procfwkHelpers].[SetDefaultTenant]; EXEC [procfwkHelpers].[SetDefaultSubscription]; EXEC [procfwkHelpers].[SetDefaultOrchestrators]; ================================================ FILE: MetadataDB/Scripts/Metadata/PipelineDependencies.sql ================================================ EXEC [procfwkHelpers].[SetDefaultPipelineDependants]; ================================================ FILE: MetadataDB/Scripts/Metadata/PipelineParams.sql ================================================ EXEC [procfwkHelpers].[SetDefaultPipelineParameters]; ================================================ FILE: MetadataDB/Scripts/Metadata/Pipelines.sql ================================================ EXEC [procfwkHelpers].[SetDefaultPipelines]; ================================================ FILE: MetadataDB/Scripts/Metadata/Properties.sql ================================================ EXEC [procfwkHelpers].[SetDefaultProperties]; ================================================ FILE: MetadataDB/Scripts/Metadata/RecipientAlertsLink.sql ================================================ EXEC [procfwkHelpers].[SetDefaultRecipientPipelineAlerts] ================================================ FILE: MetadataDB/Scripts/Metadata/Recipients.sql ================================================ EXEC [procfwkHelpers].[SetDefaultRecipients]; ================================================ FILE: MetadataDB/Scripts/Metadata/ReplaceDataFactorys.sql ================================================ IF EXISTS ( SELECT * FROM sys.objects o INNER JOIN sys.schemas s ON o.[schema_id] = s.[schema_id] WHERE o.[name] = 'DataFactorys' AND s.[name] = 'procfwk' AND o.[type] = 'U' --Check for tables as created synonyms to support backwards compatability ) BEGIN DROP TABLE [procfwk].[DataFactorys]; EXEC('CREATE VIEW [procfwk].[DataFactorys] AS SELECT [OrchestratorId] AS DataFactoryId, [OrchestratorName] AS DataFactoryName, [ResourceGroupName], [SubscriptionId], [Description] FROM [procfwk].[Orchestrators] WHERE [OrchestratorType] = ''ADF'';') END; ================================================ FILE: MetadataDB/Scripts/Metadata/Stages.sql ================================================ EXEC [procfwkHelpers].[SetDefaultBatches]; EXEC [procfwkHelpers].[SetDefaultStages]; EXEC [procfwkHelpers].[SetDefaultBatchStageLink]; ================================================ FILE: MetadataDB/Scripts/Metadata/TransferHelperObjects.sql ================================================ IF OBJECT_ID(N'tempdb..#TransferHelperObjects') IS NOT NULL DROP PROCEDURE #TransferHelperObjects; GO CREATE PROCEDURE #TransferHelperObjects ( @ObjectName NVARCHAR(128), @ObjectType CHAR(2) ) AS BEGIN IF EXISTS ( SELECT * FROM sys.objects o INNER JOIN sys.schemas s ON o.[schema_id] = s.[schema_id] WHERE o.[Name] = @ObjectName AND s.[name] = 'procfwk' AND o.[type] = @ObjectType ) BEGIN PRINT 'Transferring: ' + @ObjectName; EXEC('ALTER SCHEMA [procfwkHelpers] TRANSFER [procfwk].[' + @ObjectName + '];') END; END; GO EXEC #TransferHelperObjects 'AddProperty', 'P'; EXEC #TransferHelperObjects 'GetExecutionDetails', 'P'; EXEC #TransferHelperObjects 'AddRecipientPipelineAlerts', 'P'; EXEC #TransferHelperObjects 'DeleteRecipientAlerts', 'P'; EXEC #TransferHelperObjects 'CheckStageAndPiplineIntegrity', 'P'; EXEC #TransferHelperObjects 'AddPipelineDependant', 'P'; EXEC #TransferHelperObjects 'AddServicePrincipalWrapper', 'P'; EXEC #TransferHelperObjects 'AddServicePrincipalUrls', 'P'; EXEC #TransferHelperObjects 'AddServicePrincipal', 'P'; EXEC #TransferHelperObjects 'DeleteServicePrincipal', 'P'; EXEC #TransferHelperObjects 'CheckForValidURL', 'FN'; EXEC #TransferHelperObjects 'PipelineDependencyChains', 'V'; ================================================ FILE: MetadataDB/Scripts/Metadata/TransferReportingObjects.sql ================================================ IF OBJECT_ID(N'tempdb..#TransferReportingObjects') IS NOT NULL DROP PROCEDURE #TransferReportingObjects; GO CREATE PROCEDURE #TransferReportingObjects ( @ObjectName NVARCHAR(128), @ObjectType CHAR(2) ) AS BEGIN IF EXISTS ( SELECT * FROM sys.objects o INNER JOIN sys.schemas s ON o.[schema_id] = s.[schema_id] WHERE o.[Name] = @ObjectName AND s.[name] = 'procfwk' AND o.[type] = @ObjectType ) BEGIN PRINT 'Transferring: ' + @ObjectName; EXEC('ALTER SCHEMA [procfwkHelpers] TRANSFER [procfwk].[' + @ObjectName + '];') END; END; GO EXEC #TransferReportingObjects 'PipelineDependencyChains', 'V'; ================================================ FILE: MetadataDB/Scripts/Script.PostDeployment.sql ================================================ /* Post-Deployment Script Template -------------------------------------------------------------------------------------- This file contains SQL statements that will be appended to the build script. Use SQLCMD syntax to include a file in the post-deployment script. Example: :r .\myfile.sql Use SQLCMD syntax to reference a variable in the post-deployment script. Example: :setvar TableName MyTable SELECT * FROM [$(TableName)] -------------------------------------------------------------------------------------- */ --load default metadata :r .\Metadata\Properties.sql :r .\Metadata\Orchestrators.sql :r .\Metadata\Stages.sql :r .\Metadata\Pipelines.sql :r .\Metadata\PipelineParams.sql :r .\Metadata\PipelineDependencies.sql :r .\Metadata\Recipients.sql :r .\Metadata\AlertOutcomes.sql :r .\Metadata\RecipientAlertsLink.sql --restore log data :r .\LogData\ExecutionLogRestore.sql :r .\LogData\ErrorLogRestore.sql --object transfers :r .\Metadata\TransferHelperObjects.sql :r .\Metadata\TransferReportingObjects.sql --replace old objects :r .\Metadata\ReplaceDataFactorys.sql ================================================ FILE: MetadataDB/Scripts/Script.PreDeployment.sql ================================================ /* Pre-Deployment Script Template -------------------------------------------------------------------------------------- This file contains SQL statements that will be executed before the build script. Use SQLCMD syntax to include a file in the pre-deployment script. Example: :r .\myfile.sql Use SQLCMD syntax to reference a variable in the pre-deployment script. Example: :setvar TableName MyTable SELECT * FROM [$(TableName)] -------------------------------------------------------------------------------------- */ --data :r .\LogData\ExecutionLogBackup.sql :r .\LogData\ErrorLogBackup.sql --delete all :r .\Metadata\DropLegacyTables.sql :r .\Metadata\DropLegacyObjects.sql :r .\Metadata\DeleteAll.sql ================================================ FILE: MetadataDB/Scripts/Script.SetLocalAuthenticationDetails.sql ================================================ --Run in SQLCMD mode to use environment variables. --Optional script used to setup function testing for processing framework parts post deployment. PRINT 'TenantId: ' + '$(AZURE_TENANT_ID)' PRINT 'SubscriptionId: ' + '$(AZURE_SUBSCRIPTION_ID)' --add my tenant INSERT INTO [procfwk].[Tenants] ( [TenantId], [Name], [Description] ) VALUES ('$(AZURE_TENANT_ID)', 'mrpaulandrew.com', NULL); --add my subscription INSERT INTO [procfwk].[Subscriptions] ( [SubscriptionId], [Name], [Description], [TenantId] ) VALUES ('$(AZURE_SUBSCRIPTION_ID)', 'Microsoft Sponsored Subscription', NULL, '$(AZURE_TENANT_ID)'); --update Orchestrator with new subscription UPDATE [procfwk].[Orchestrators] SET [SubscriptionId] = '$(AZURE_SUBSCRIPTION_ID)'; --remove default values DELETE FROM [procfwk].[Subscriptions] WHERE [SubscriptionId] = '12345678-1234-1234-1234-012345678910'; DELETE FROM [procfwk].[Tenants] WHERE [TenantId] = '12345678-1234-1234-1234-012345678910'; /* EXEC [procfwkHelpers].[AddProperty] @PropertyName = N'SPNHandlingMethod', @PropertyValue = N'StoreInKeyVault', @Description = N'Accepted values: StoreInDatabase, StoreInKeyVault. See v1.8.2 release notes for full details.'; */ IF (SELECT [procfwk].[GetPropertyValueInternal]('SPNHandlingMethod')) = 'StoreInDatabase' BEGIN --Add SPN for execution of all worker pipelines using database to store SPN values EXEC [procfwkHelpers].[AddServicePrincipalWrapper] @OrchestratorName = N'FrameworkFactory', @OrchestratorType = 'ADF', @PrincipalIdValue = '$(AZURE_CLIENT_ID)', @PrincipalSecretValue = '$(AZURE_CLIENT_SECRET)', @PrincipalName = '$(AZURE_CLIENT_NAME)'; EXEC [procfwkHelpers].[AddServicePrincipalWrapper] @OrchestratorName = N'procfwkforsynapse', @OrchestratorType = 'SYN', @PrincipalIdValue = '$(AZURE_CLIENT_ID)', @PrincipalSecretValue = '$(AZURE_CLIENT_SECRET)', @PrincipalName = '$(AZURE_CLIENT_NAME)'; /* --Add specific SPN for execution of Wait 1 pipeline (functional testing) EXEC [procfwkHelpers].[DeleteServicePrincipal] @OrchestratorName = N'FrameworkFactory', @OrchestratorType = 'ADF', @PrincipalIdValue = '$(AZURE_CLIENT_ID)', @SpecificPipelineName = N'Wait 1'; EXEC [procfwkHelpers].[AddServicePrincipalWrapper] @OrchestratorName = N'FrameworkFactory', @OrchestratorType = 'ADF', @PrincipalIdValue = '$(AZURE_CLIENT_ID_2)', @PrincipalSecretValue = '$(AZURE_CLIENT_SECRET_2)', @PrincipalName = '$(AZURE_CLIENT_NAME_2)', @SpecificPipelineName = N'Wait 1'; */ END ELSE IF (SELECT [procfwk].[GetPropertyValueInternal]('SPNHandlingMethod')) = 'StoreInKeyVault' BEGIN --Add SPN for execution of all worker pipelines using database to store key vault URLs EXEC [procfwkHelpers].[AddServicePrincipalWrapper] @OrchestratorName = N'FrameworkFactory', @OrchestratorType = 'ADF', @PrincipalIdValue = '$(AZURE_CLIENT_ID_URL)', @PrincipalSecretValue = '$(AZURE_CLIENT_SECRET_URL)', @PrincipalName = '$(AZURE_CLIENT_NAME)'; END ELSE BEGIN RAISERROR('Unknown SPN insert method.',16,1); END; SELECT * FROM [dbo].[ServicePrincipals]; ================================================ FILE: MetadataDB/Security/procfwk.sql ================================================ CREATE SCHEMA [procfwk] AUTHORIZATION [dbo]; ================================================ FILE: MetadataDB/Security/procfwkHelpers.sql ================================================ CREATE SCHEMA [procfwkHelpers] AUTHORIZATION [dbo]; ================================================ FILE: MetadataDB/Security/procfwkReporting.sql ================================================ CREATE SCHEMA [procfwkReporting] AUTHORIZATION [dbo]; ================================================ FILE: MetadataDB/Security/procfwkTesting.sql ================================================ CREATE SCHEMA [procfwkTesting] AUTHORIZATION [dbo]; ================================================ FILE: MetadataDB/Security/procfwkuser Role.sql ================================================ /* CREATE USER [##Data Factory Name (Managed Identity)##] FROM EXTERNAL PROVIDER; */ CREATE ROLE [procfwkuser] GO GRANT EXECUTE, SELECT, CONTROL, ALTER ON SCHEMA::[procfwk] TO [procfwkuser] GO /* ALTER ROLE [procfwkuser] ADD MEMBER [##Data Factory Name (Managed Identity)##]; */ ================================================ FILE: MetadataDB/dbo/Stored Procedures/DemoModePrecursor.sql ================================================ CREATE PROCEDURE [dbo].[DemoModePrecursor] AS BEGIN --quick win IF ([procfwk].[GetPropertyValueInternal]('ExecutionPrecursorProc')) <> '[dbo].[DemoModePrecursor]' BEGIN EXEC [procfwkHelpers].[AddProperty] @PropertyName = N'ExecutionPrecursorProc', @PropertyValue = N'[dbo].[DemoModePrecursor]'; END; --reduce wait times ;WITH cte AS ( SELECT [PipelineId], LEFT(ABS(CAST(CAST(NEWID() AS VARBINARY(192)) AS INT)),1) AS NewValue FROM [procfwk].[PipelineParameters] ) UPDATE pp SET pp.[ParameterValue] = cte.[NewValue] FROM [procfwk].[PipelineParameters] pp INNER JOIN cte ON pp.[PipelineId] = cte.[PipelineId] INNER JOIN [procfwk].[Pipelines] p ON pp.[PipelineId] = p.[PipelineId] WHERE pp.[ParameterName] LIKE 'Wait%' AND p.[Enabled] = 1; --for intentional error IF NOT EXISTS ( SELECT * FROM [procfwk].[CurrentExecution] ) BEGIN UPDATE pp SET pp.[ParameterValue] = 'true' FROM [procfwk].[PipelineParameters] pp INNER JOIN [procfwk].[Pipelines] p ON pp.[PipelineId] = p.[PipelineId] WHERE p.[PipelineName] = 'Intentional Error' AND pp.[ParameterName] = 'RaiseErrors'; END; ELSE BEGIN UPDATE pp SET pp.[ParameterValue] = 'false' FROM [procfwk].[PipelineParameters] pp INNER JOIN [procfwk].[Pipelines] p ON pp.[PipelineId] = p.[PipelineId] WHERE p.[PipelineName] = 'Intentional Error' AND pp.[ParameterName] = 'RaiseErrors'; END; --dependency chain failure handling IF ([procfwk].[GetPropertyValueInternal]('FailureHandling')) <> 'DependencyChain' BEGIN EXEC [procfwkHelpers].[AddProperty] @PropertyName = N'FailureHandling', @PropertyValue = N'DependencyChain'; END; --short infant iterations IF ([procfwk].[GetPropertyValueInternal]('PipelineStatusCheckDuration')) <> '5' BEGIN EXEC [procfwkHelpers].[AddProperty] @PropertyName = N'PipelineStatusCheckDuration', @PropertyValue = N'5'; END; END; ================================================ FILE: MetadataDB/dbo/Stored Procedures/ExampleCustomExecutionPrecursor.sql ================================================ CREATE PROCEDURE [dbo].[ExampleCustomExecutionPrecursor] AS BEGIN --set random Worker pipeline parameter wait times for development environment ;WITH cte AS ( SELECT [PipelineId], LEFT(ABS(CAST(CAST(NEWID() AS VARBINARY(192)) AS INT)),2) AS NewValue FROM [procfwk].[PipelineParameters] ) UPDATE pp SET pp.[ParameterValue] = cte.[NewValue] FROM [procfwk].[PipelineParameters] pp INNER JOIN cte ON pp.[PipelineId] = cte.[PipelineId] INNER JOIN [procfwk].[Pipelines] p ON pp.[PipelineId] = p.[PipelineId] WHERE pp.[ParameterName] LIKE 'Wait%' AND p.[Enabled] = 1; --disable certain Workers if running at the weekend... -- YOUR CODE HERE --enable certain Workers if running on the 10th day of the month... -- YOUR CODE HERE --disable certain Stages if running on Friday... -- YOUR CODE HERE --set Worker pipeline parameters to new value based on ______ .... -- YOUR CODE HERE --etc END; ================================================ FILE: MetadataDB/dbo/Stored Procedures/FailProcedure.sql ================================================ CREATE PROCEDURE [dbo].[FailProcedure] ( @RaiseError VARCHAR(50) ) AS BEGIN IF(@RaiseError = 'true') BEGIN RAISERROR('The Stored Procedure intentionally failed.',16,1); RETURN 0; END END; ================================================ FILE: MetadataDB/dbo/Tables/ServicePrincipals.sql ================================================ CREATE TABLE [dbo].[ServicePrincipals]( [CredentialId] INT IDENTITY(1,1) NOT NULL, [PrincipalName] NVARCHAR(256) NULL, [PrincipalId] UNIQUEIDENTIFIER NULL, [PrincipalSecret] VARBINARY(256) NULL, [PrincipalIdUrl] NVARCHAR(MAX) NULL, [PrincipalSecretUrl] NVARCHAR(MAX) NULL, CONSTRAINT [PK_ServicePrincipals] PRIMARY KEY CLUSTERED ([CredentialId] ASC) ) GO ================================================ FILE: MetadataDB/procfwk/Functions/GetPropertyValueInternal.sql ================================================ CREATE FUNCTION [procfwk].[GetPropertyValueInternal] ( @PropertyName VARCHAR(128) ) RETURNS NVARCHAR(MAX) AS BEGIN DECLARE @PropertyValue NVARCHAR(MAX) SELECT @PropertyValue = ISNULL([PropertyValue],'') FROM [procfwk].[CurrentProperties] WHERE [PropertyName] = @PropertyName RETURN @PropertyValue END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/BatchWrapper.sql ================================================ CREATE PROCEDURE [procfwk].[BatchWrapper] ( @BatchId UNIQUEIDENTIFIER, @LocalExecutionId UNIQUEIDENTIFIER OUTPUT ) AS BEGIN SET NOCOUNT ON; DECLARE @RestartStatus BIT --get restart overide property SELECT @RestartStatus = [procfwk].[GetPropertyValueInternal]('OverideRestart') --check for running batch execution IF EXISTS ( SELECT 1 FROM [procfwk].[BatchExecution] WHERE [BatchId] = @BatchId AND ISNULL([BatchStatus],'') = 'Running' ) BEGIN SELECT @LocalExecutionId = [ExecutionId] FROM [procfwk].[BatchExecution] WHERE [BatchId] = @BatchId; --should never actually be called as handled within Orchestrator pipelines using the Pipeline Run Check utility RAISERROR('There is already an batch execution run in progress. Stop the related parent pipeline via the Orchestrator first.',16,1); RETURN 0; END ELSE IF EXISTS ( SELECT 1 FROM [procfwk].[BatchExecution] WHERE [BatchId] = @BatchId AND ISNULL([BatchStatus],'') = 'Stopped' ) AND @RestartStatus = 0 BEGIN SELECT @LocalExecutionId = [ExecutionId] FROM [procfwk].[BatchExecution] WHERE [BatchId] = @BatchId AND ISNULL([BatchStatus],'') = 'Stopped'; RETURN 0; END ELSE IF EXISTS ( SELECT 1 FROM [procfwk].[BatchExecution] WHERE [BatchId] = @BatchId AND ISNULL([BatchStatus],'') = 'Stopped' ) AND @RestartStatus = 1 BEGIN --clean up current execution table and abandon batch SELECT @LocalExecutionId = [ExecutionId] FROM [procfwk].[BatchExecution] WHERE [BatchId] = @BatchId AND ISNULL([BatchStatus],'') = 'Stopped'; EXEC [procfwk].[UpdateExecutionLog] @PerformErrorCheck = 0, --Special case when OverideRestart = 1; @ExecutionId = @LocalExecutionId; --abandon previous batch execution UPDATE [procfwk].[BatchExecution] SET [BatchStatus] = 'Abandoned' WHERE [BatchId] = @BatchId AND ISNULL([BatchStatus],'') = 'Stopped'; SET @LocalExecutionId = NEWID(); --create new batch run record INSERT INTO [procfwk].[BatchExecution] ( [BatchId], [ExecutionId], [BatchName], [BatchStatus], [StartDateTime] ) SELECT [BatchId], @LocalExecutionId, [BatchName], 'Running', GETUTCDATE() FROM [procfwk].[Batches] WHERE [BatchId] = @BatchId; END ELSE BEGIN SET @LocalExecutionId = NEWID(); --create new batch run record INSERT INTO [procfwk].[BatchExecution] ( [BatchId], [ExecutionId], [BatchName], [BatchStatus], [StartDateTime] ) SELECT [BatchId], @LocalExecutionId, [BatchName], 'Running', GETUTCDATE() FROM [procfwk].[Batches] WHERE [BatchId] = @BatchId; END; END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/CheckForBlockedPipelines.sql ================================================ CREATE PROCEDURE [procfwk].[CheckForBlockedPipelines] ( @ExecutionId UNIQUEIDENTIFIER, @StageId INT ) AS BEGIN SET NOCOUNT ON; -- If any pipelines still have a status of running, mark as failed to block downstream processing, and add an error log IF EXISTS ( SELECT * FROM [procfwk].[CurrentExecution] WHERE [LocalExecutionId] = @ExecutionId AND [StageId] < @StageId AND [PipelineStatus] = 'Running' ) BEGIN DECLARE @RunningPipelineId INT; DECLARE @RunningPipelineStageId INT; DECLARE @RunId UNIQUEIDENTIFIER; DECLARE @ErrorJson NVARCHAR(MAX); DECLARE @RunningCursor CURSOR ; SET @RunningCursor = CURSOR FOR SELECT [PipelineId], [StageId], [PipelineRunId] FROM [procfwk].[CurrentExecution] WHERE [LocalExecutionId] = @ExecutionId AND [StageId] < @StageId AND [PipelineStatus] = 'Running' OPEN @RunningCursor FETCH NEXT FROM @RunningCursor INTO @RunningPipelineId, @RunningPipelineStageId, @RunId WHILE @@FETCH_STATUS = 0 BEGIN EXEC [procfwk].SetLogPipelineFailed @ExecutionId = @ExecutionId, @StageId = @RunningPipelineStageId, @PipelineId = @RunningPipelineId, @RunId = @RunId; SET @ErrorJson = '{ "RunId": "' + Cast(@RunId AS CHAR(36)) + '", "Errors": [ { "ActivityRunId": "00000000-0000-0000-0000-000000000000", "ActivityName": "Set Pipeline Result", "ActivityType": "Switch", "ErrorCode": "Unknown", "ErrorType": "Framework Error", "ErrorMessage": "Framework pipeline ''04-Infant'' failed to set the pipeline result, most likely due to a timeout or azure connectivity issue. Check the framework Data Factory monitor for more information." } ] }' EXEC procfwk.SetErrorLogDetails @LocalExecutionId = @ExecutionId, @JsonErrorDetails = @ErrorJson; FETCH NEXT FROM @RunningCursor INTO @RunningPipelineId, @RunningPipelineStageId, @RunId; END; CLOSE @RunningCursor; DEALLOCATE @RunningCursor; END; IF ([procfwk].[GetPropertyValueInternal]('FailureHandling')) = 'None' BEGIN --do nothing allow processing to carry on regardless RETURN 0; END; ELSE IF ([procfwk].[GetPropertyValueInternal]('FailureHandling')) = 'Simple' BEGIN IF EXISTS ( SELECT * FROM [procfwk].[CurrentExecution] WHERE [LocalExecutionId] = @ExecutionId AND [StageId] = @StageId AND [IsBlocked] = 1 ) BEGIN UPDATE [procfwk].[BatchExecution] SET [EndDateTime] = GETUTCDATE(), [BatchStatus] = 'Stopped' WHERE [ExecutionId] = @ExecutionId; --Saves the infant pipeline and activities being called throwing the exception at this level. RAISERROR('All pipelines are blocked. Stopping processing.',16,1); --If not thrown here, the proc [procfwk].[UpdateExecutionLog] would eventually throw an exception. RETURN 0; END END; ELSE IF ([procfwk].[GetPropertyValueInternal]('FailureHandling')) = 'DependencyChain' BEGIN IF EXISTS ( SELECT * FROM [procfwk].[CurrentExecution] WHERE [LocalExecutionId] = @ExecutionId AND [StageId] = @StageId AND [IsBlocked] = 1 ) BEGIN DECLARE @PipelineId INT; DECLARE @Cursor CURSOR ; SET @Cursor = CURSOR FOR SELECT [PipelineId] FROM [procfwk].[CurrentExecution] WHERE [LocalExecutionId] = @ExecutionId AND [StageId] = @StageId AND [IsBlocked] = 1 OPEN @Cursor FETCH NEXT FROM @Cursor INTO @PipelineId WHILE @@FETCH_STATUS = 0 BEGIN EXEC [procfwk].[SetExecutionBlockDependants] @ExecutionId = @ExecutionId, @PipelineId = @PipelineId; FETCH NEXT FROM @Cursor INTO @PipelineId; END; CLOSE @Cursor; DEALLOCATE @Cursor; END; END; ELSE BEGIN RAISERROR('Unknown failure handling state.',16,1); RETURN 0; END; END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/CheckForEmailAlerts.sql ================================================ CREATE PROCEDURE [procfwk].[CheckForEmailAlerts] ( @PipelineId INT ) AS BEGIN SET NOCOUNT ON; DECLARE @SendAlerts BIT DECLARE @AlertingEnabled BIT --get property SELECT @AlertingEnabled = [procfwk].[GetPropertyValueInternal]('UseFrameworkEmailAlerting'); --based on global property IF (@AlertingEnabled = 1) BEGIN --based on piplines to recipients link IF EXISTS ( SELECT pal.AlertId FROM procfwk.CurrentExecution AS ce INNER JOIN procfwk.AlertOutcomes AS ao ON ao.PipelineOutcomeStatus = ce.PipelineStatus INNER JOIN procfwk.PipelineAlertLink AS pal ON pal.PipelineId = ce.PipelineId INNER JOIN procfwk.Recipients AS r ON r.RecipientId = pal.RecipientId WHERE ce.PipelineId = @PipelineId AND ( ao.BitValue & pal.OutcomesBitValue <> 0 OR pal.OutcomesBitValue & 1 <> 0 --all ) AND pal.[Enabled] = 1 AND r.[Enabled] = 1 ) BEGIN SET @SendAlerts = 1; END; ELSE BEGIN SET @SendAlerts = 0; END; END ELSE BEGIN SET @SendAlerts = 0; END; SELECT @SendAlerts AS SendAlerts END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/CheckMetadataIntegrity.sql ================================================ CREATE PROCEDURE [procfwk].[CheckMetadataIntegrity] ( @DebugMode BIT = 0, @BatchName VARCHAR(255) = NULL ) AS BEGIN SET NOCOUNT ON; /* Check 1 - Are there execution stages enabled in the metadata? Check 2 - Are there pipelines enabled in the metadata? Check 3 - Are there any service principals available to run the processing pipelines? Check 4 - Is there at least one TenantId available? Check 5 - Is there at least one SubscriptionId available? Check 6 - Is there a current OverideRestart property available? Check 7 - Are there any enabled pipelines configured without a service principal? Check 8 - Are any Orchestrators set to use the default subscription value? Check 9 - Are any Subscription set to use the default tenant value? Check 10 - Is there a current PipelineStatusCheckDuration property available? Check 11 - Is there a current UseFrameworkEmailAlerting property available? Check 12 - Is there a current EmailAlertBodyTemplate property available? Check 13 - Does the total size of the request body for the pipeline parameters added exceed the Azure Functions size limit when the Worker execute pipeline body is created? Check 14 - Is there a current FailureHandling property available? Check 15 - Does the FailureHandling property have a valid value? Check 16 - When using DependencyChain failure handling, are there any dependants in the same execution stage of the predecessor? Check 17 - Does the SPNHandlingMethod property have a valid value? Check 18 - Does the Service Principal table contain both types of SPN handling for a single credential? Check 19 - Is there a current UseExecutionBatches property available? Check 20 - Is there a current FrameworkFactoryResourceGroup property available? Check 21 - Is there a current PreviousPipelineRunsQueryRange property available? --Batch execution checks: Check 22 - If using batch executions, is the requested batch name enabled? Check 23 - If using batch executions, does the requested batch have links to execution stages? Check 24 - Have batch executions been enabled after a none batch execution run? Check 25 - Has the execution failed due to an invalid pipeline name? If so, attend to update this before the next run. Check 26 - Is there more than one framework orchestrator set? Check 27 - Has a framework orchestrator been set for any orchestrators? */ DECLARE @BatchId UNIQUEIDENTIFIER DECLARE @ErrorDetails VARCHAR(500) DECLARE @MetadataIntegrityIssues TABLE ( [CheckNumber] INT NOT NULL, [IssuesFound] VARCHAR(MAX) NOT NULL ) /* Checks: */ --Check 1: IF NOT EXISTS ( SELECT 1 FROM [procfwk].[Stages] WHERE [Enabled] = 1 ) BEGIN INSERT INTO @MetadataIntegrityIssues VALUES ( 1, 'No execution stages are enabled within the metadatabase. Orchestrator has nothing to run.' ) END; --Check 2: IF NOT EXISTS ( SELECT 1 FROM [procfwk].[Pipelines] WHERE [Enabled] = 1 ) BEGIN INSERT INTO @MetadataIntegrityIssues VALUES ( 2, 'No execution pipelines are enabled within the metadatabase. Orchestrator has nothing to run.' ) END; --Check 3: IF NOT EXISTS ( SELECT 1 FROM [dbo].[ServicePrincipals] ) BEGIN INSERT INTO @MetadataIntegrityIssues VALUES ( 3, 'No service principal details have been added to the metadata. Orchestrator cannot authorise pipeline executions.' ) END; --Check 4: IF NOT EXISTS ( SELECT * FROM [procfwk].[Tenants] ) BEGIN INSERT INTO @MetadataIntegrityIssues VALUES ( 4, 'TenantId value is missing from the [procfwk].[Tenants] table.' ) END; --Check 5: IF NOT EXISTS ( SELECT * FROM [procfwk].[Subscriptions] ) BEGIN INSERT INTO @MetadataIntegrityIssues VALUES ( 5, 'SubscriptionId value is missing from the [procfwk].[Subscriptions] table.' ) END; --Check 6: IF NOT EXISTS ( SELECT * FROM [procfwk].[CurrentProperties] WHERE [PropertyName] = 'OverideRestart' ) BEGIN INSERT INTO @MetadataIntegrityIssues VALUES ( 6, 'A current OverideRestart value is missing from the properties table.' ) END; --Check 7: IF EXISTS ( SELECT * FROM [procfwk].[Pipelines] p LEFT OUTER JOIN [procfwk].[PipelineAuthLink] al ON p.[PipelineId] = al.[PipelineId] WHERE p.[Enabled] = 1 AND al.[PipelineId] IS NULL ) BEGIN INSERT INTO @MetadataIntegrityIssues VALUES ( 7, 'Enabled pipelines are missing a valid Service Principal link.' ) END; --Check 8: IF EXISTS ( SELECT * FROM [procfwk].[Orchestrators] WHERE [SubscriptionId] = '12345678-1234-1234-1234-012345678910' ) BEGIN INSERT INTO @MetadataIntegrityIssues VALUES ( 8, 'Orchestrator still set to use the default subscription value of 12345678-1234-1234-1234-012345678910.' ) END; --Check 9: IF EXISTS ( SELECT * FROM [procfwk].[Subscriptions] WHERE [TenantId] = '12345678-1234-1234-1234-012345678910' AND [SubscriptionId] <> '12345678-1234-1234-1234-012345678910' ) BEGIN INSERT INTO @MetadataIntegrityIssues VALUES ( 9, 'None default subscription still set to use the default tenant value of 12345678-1234-1234-1234-012345678910.' ) END; --Check 10: IF NOT EXISTS ( SELECT * FROM [procfwk].[CurrentProperties] WHERE [PropertyName] = 'PipelineStatusCheckDuration' ) BEGIN INSERT INTO @MetadataIntegrityIssues VALUES ( 10, 'A current PipelineStatusCheckDuration value is missing from the properties table.' ) END; --Check 11: IF NOT EXISTS ( SELECT * FROM [procfwk].[CurrentProperties] WHERE [PropertyName] = 'UseFrameworkEmailAlerting' ) BEGIN INSERT INTO @MetadataIntegrityIssues VALUES ( 11, 'A current UseFrameworkEmailAlerting value is missing from the properties table.' ) END; --Check 12: IF ( SELECT [PropertyValue] FROM [procfwk].[CurrentProperties] WHERE [PropertyName] = 'UseFrameworkEmailAlerting' ) = 1 BEGIN IF NOT EXISTS ( SELECT * FROM [procfwk].[CurrentProperties] WHERE [PropertyName] = 'EmailAlertBodyTemplate' ) BEGIN INSERT INTO @MetadataIntegrityIssues VALUES ( 12, 'A current EmailAlertBodyTemplate value is missing from the properties table.' ) END; END; --Check 13: IF EXISTS ( SELECT * FROM [procfwk].[PipelineParameterDataSizes] WHERE [Size] > 9 /* Azure Function request limit is 10MB. https://docs.microsoft.com/en-us/azure/azure-functions/functions-scale 9MB to allow for other content in execute pipeline body request. */ ) BEGIN INSERT INTO @MetadataIntegrityIssues VALUES ( 13, 'The pipeline parameters entered exceed the Azure Function request body maximum of 10MB. Query view [procfwk].[PipelineParameterDataSizes] for details.' ) END; --Check 14: IF NOT EXISTS ( SELECT * FROM [procfwk].[CurrentProperties] WHERE [PropertyName] = 'FailureHandling' ) BEGIN INSERT INTO @MetadataIntegrityIssues VALUES ( 14, 'A current FailureHandling value is missing from the properties table.' ) END; --Check 15: IF NOT EXISTS ( SELECT * FROM [procfwk].[CurrentProperties] WHERE [PropertyName] = 'FailureHandling' AND [PropertyValue] IN ('None','Simple','DependencyChain') ) BEGIN INSERT INTO @MetadataIntegrityIssues VALUES ( 15, 'The property FailureHandling does not have a supported value.' ) END; --Check 16: IF ([procfwk].[GetPropertyValueInternal]('FailureHandling')) = 'DependencyChain' BEGIN IF EXISTS ( SELECT pd.[DependencyId] FROM [procfwk].[PipelineDependencies] pd INNER JOIN [procfwk].[Pipelines] pp ON pd.[PipelineId] = pp.[PipelineId] INNER JOIN [procfwk].[Pipelines] dp ON pd.[DependantPipelineId] = dp.[PipelineId] WHERE pp.[StageId] = dp.[StageId] ) BEGIN INSERT INTO @MetadataIntegrityIssues VALUES ( 16, 'A dependant pipeline and its upstream predecessor exist in the same execution stage. Fix this dependency chain to allow correct failure handling.' ) END; END; --Check 17: IF NOT EXISTS ( SELECT * FROM [procfwk].[CurrentProperties] WHERE [PropertyName] = 'SPNHandlingMethod' AND [PropertyValue] IN ('StoreInDatabase','StoreInKeyVault') ) BEGIN INSERT INTO @MetadataIntegrityIssues VALUES ( 17, 'The property SPNHandlingMethod does not have a supported value.' ) END; --Check 18: IF EXISTS ( SELECT * FROM [dbo].[ServicePrincipals] WHERE ( [PrincipalId] IS NOT NULL OR [PrincipalSecret] IS NOT NULL ) AND ( [PrincipalIdUrl] IS NOT NULL OR [PrincipalSecretUrl] IS NOT NULL ) ) BEGIN INSERT INTO @MetadataIntegrityIssues VALUES ( 18, 'The table [dbo].[ServicePrincipals] can only have one method of SPN details sorted per credential ID.' ) END; --Check 19: IF NOT EXISTS ( SELECT * FROM [procfwk].[CurrentProperties] WHERE [PropertyName] = 'UseExecutionBatches' ) BEGIN INSERT INTO @MetadataIntegrityIssues VALUES ( 19, 'A current UseExecutionBatches value is missing from the properties table.' ) END; --Check 20: IF NOT EXISTS ( SELECT * FROM [procfwk].[CurrentProperties] WHERE [PropertyName] = 'FrameworkFactoryResourceGroup' ) BEGIN INSERT INTO @MetadataIntegrityIssues VALUES ( 20, 'A current FrameworkFactoryResourceGroup value is missing from the properties table.' ) END; --Check 21: IF NOT EXISTS ( SELECT * FROM [procfwk].[CurrentProperties] WHERE [PropertyName] = 'PreviousPipelineRunsQueryRange' ) BEGIN INSERT INTO @MetadataIntegrityIssues VALUES ( 21, 'A current PreviousPipelineRunsQueryRange value is missing from the properties table.' ) END; --batch execution checks IF ([procfwk].[GetPropertyValueInternal]('UseExecutionBatches')) = '1' BEGIN IF @BatchName IS NULL BEGIN RAISERROR('A NULL batch name cannot be passed when the UseExecutionBatches property is set to 1 (true).',16,1); RETURN 0; END SELECT @BatchId = [BatchId] FROM [procfwk].[Batches] WHERE [BatchName] = @BatchName; --Check 22: IF EXISTS ( SELECT 1 FROM [procfwk].[Batches] WHERE [BatchId] = @BatchId AND [Enabled] = 0 ) BEGIN INSERT INTO @MetadataIntegrityIssues VALUES ( 22, 'The requested execution batch is currently disabled. Enable the batch before proceeding.' ) END; --Check 23: IF NOT EXISTS ( SELECT 1 FROM [procfwk].[BatchStageLink] WHERE [BatchId] = @BatchId ) BEGIN INSERT INTO @MetadataIntegrityIssues VALUES ( 23, 'The requested execution batch does not have any linked execution stages. See table [procfwk].[BatchStageLink] for details.' ) END; --Check 24: IF EXISTS ( SELECT * FROM [procfwk].[CurrentExecution] c LEFT OUTER JOIN [procfwk].[BatchExecution] b ON c.[LocalExecutionId] = b.[ExecutionId] WHERE b.[ExecutionId] IS NULL ) BEGIN INSERT INTO @MetadataIntegrityIssues VALUES ( 24, 'Execution records exist in the [procfwk].[CurrentExecution] table that do not have a record in [procfwk].[BatchExecution] table. Has batch excutions been enabed after an incomplete none batch run?' ) END; END; --end batch checks --Check 25: IF EXISTS ( SELECT 1 FROM [procfwk].[CurrentExecution] WHERE [PipelineStatus] = 'InvalidPipelineNameError' ) BEGIN UPDATE ce SET ce.[PipelineName] = p.[PipelineName] FROM [procfwk].[CurrentExecution] ce INNER JOIN [procfwk].[Pipelines] p ON ce.[PipelineId] = p.[PipelineId] AND ce.[StageId] = p.[StageId] WHERE ce.[PipelineStatus] = 'InvalidPipelineNameError' END; --Check 26: IF (SELECT COUNT(0) FROM [procfwk].[Orchestrators] WHERE [IsFrameworkOrchestrator] = 1) > 1 BEGIN INSERT INTO @MetadataIntegrityIssues VALUES ( 26, 'There is more than one FrameworkOrchestrator set in the table [procfwk].[Orchestrators]. Only one is supported.' ) END --Check 27: IF NOT EXISTS ( SELECT 1 FROM [procfwk].[Orchestrators] WHERE [IsFrameworkOrchestrator] = 1 ) BEGIN INSERT INTO @MetadataIntegrityIssues VALUES ( 27, 'A FrameworkOrchestrator has not been set in the table [procfwk].[Orchestrators]. Only one is supported.' ) END /* Integrity Checks Outcome: */ --throw runtime error if checks fail IF EXISTS ( SELECT * FROM @MetadataIntegrityIssues ) AND @DebugMode = 0 BEGIN SET @ErrorDetails = 'Metadata integrity checks failed. Run EXEC [procfwk].[CheckMetadataIntegrity] @DebugMode = 1; for details.' RAISERROR(@ErrorDetails, 16, 1); RETURN 0; END; --report issues when in debug mode IF @DebugMode = 1 BEGIN IF NOT EXISTS ( SELECT * FROM @MetadataIntegrityIssues ) BEGIN PRINT 'No data integrity issues found in metadata.' RETURN 0; END ELSE BEGIN SELECT * FROM @MetadataIntegrityIssues; END; END; END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/CheckPreviousExeuction.sql ================================================ CREATE PROCEDURE [procfwk].[CheckPreviousExeuction] ( @BatchName VARCHAR(255) = NULL ) AS BEGIN SET NOCOUNT ON; /* Check A: - Are there any Running pipelines that need to be cleaned up? */ DECLARE @BatchId UNIQUEIDENTIFIER DECLARE @LocalExecutionId UNIQUEIDENTIFIER --Check A: IF ([procfwk].[GetPropertyValueInternal]('UseExecutionBatches')) = '0' BEGIN IF EXISTS ( SELECT 1 FROM [procfwk].[CurrentExecution] WHERE [PipelineStatus] NOT IN ('Success','Failed','Blocked', 'Cancelled') AND [PipelineRunId] IS NOT NULL ) BEGIN --return pipelines details that require a clean up SELECT [ResourceGroupName], [OrchestratorType], [OrchestratorName], [PipelineName], [PipelineRunId], [LocalExecutionId], [StageId], [PipelineId] FROM [procfwk].[CurrentExecution] WHERE [PipelineStatus] NOT IN ('Success','Failed','Blocked','Cancelled') AND [PipelineRunId] IS NOT NULL END; ELSE GOTO LookUpReturnEmptyResult; END ELSE IF ([procfwk].[GetPropertyValueInternal]('UseExecutionBatches')) = '1' BEGIN IF @BatchName IS NULL BEGIN RAISERROR('A NULL batch name cannot be passed when the UseExecutionBatches property is set to 1 (true).',16,1); RETURN 0; END IF EXISTS ( SELECT 1 FROM [procfwk].[CurrentExecution] ce INNER JOIN [procfwk].[BatchExecution] be ON ce.[LocalExecutionId] = be.[ExecutionId] INNER JOIN [procfwk].[Batches] b ON be.[BatchId] = b.[BatchId] WHERE b.[BatchName] = @BatchName AND ce.[PipelineStatus] NOT IN ('Success','Failed','Blocked','Cancelled') AND ce.[PipelineRunId] IS NOT NULL ) BEGIN --return pipelines details that require a clean up SELECT ce.[ResourceGroupName], ce.[OrchestratorType], ce.[OrchestratorName], ce.[PipelineName], ce.[PipelineRunId], ce.[LocalExecutionId], ce.[StageId], ce.[PipelineId] FROM [procfwk].[CurrentExecution] ce INNER JOIN [procfwk].[BatchExecution] be ON ce.[LocalExecutionId] = be.[ExecutionId] INNER JOIN [procfwk].[Batches] b ON be.[BatchId] = b.[BatchId] WHERE b.[BatchName] = @BatchName AND ce.[PipelineStatus] NOT IN ('Success','Failed','Blocked','Cancelled') AND ce.[PipelineRunId] IS NOT NULL END; ELSE GOTO LookUpReturnEmptyResult; END LookUpReturnEmptyResult: --lookup activity must return something, even if just an empty dataset SELECT NULL AS ResourceGroupName, NULL AS OrchestratorType, NULL AS OrchestratorName, NULL AS PipelineName, NULL AS PipelineRunId, NULL AS LocalExecutionId, NULL AS StageId, NULL AS PipelineId FROM [procfwk].[CurrentExecution] WHERE 1 = 2; --ensure no results END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/CreateNewExecution.sql ================================================ CREATE PROCEDURE [procfwk].[CreateNewExecution] ( @CallingOrchestratorName NVARCHAR(200), @LocalExecutionId UNIQUEIDENTIFIER = NULL ) AS BEGIN SET NOCOUNT ON; DECLARE @BatchId UNIQUEIDENTIFIER; IF([procfwk].[GetPropertyValueInternal]('UseExecutionBatches')) = '0' BEGIN SET @LocalExecutionId = NEWID(); TRUNCATE TABLE [procfwk].[CurrentExecution]; --defensive check IF NOT EXISTS ( SELECT 1 FROM [procfwk].[Pipelines] p INNER JOIN [procfwk].[Stages] s ON p.[StageId] = s.[StageId] INNER JOIN [procfwk].[Orchestrators] d ON p.[OrchestratorId] = d.[OrchestratorId] WHERE p.[Enabled] = 1 AND s.[Enabled] = 1 ) BEGIN RAISERROR('Requested execution run does not contain any enabled stages/pipelines.',16,1); RETURN 0; END; INSERT INTO [procfwk].[CurrentExecution] ( [LocalExecutionId], [StageId], [PipelineId], [CallingOrchestratorName], [ResourceGroupName], [OrchestratorType], [OrchestratorName], [PipelineName] ) SELECT @LocalExecutionId, p.[StageId], p.[PipelineId], @CallingOrchestratorName, d.[ResourceGroupName], d.[OrchestratorType], d.[OrchestratorName], p.[PipelineName] FROM [procfwk].[Pipelines] p INNER JOIN [procfwk].[Stages] s ON p.[StageId] = s.[StageId] INNER JOIN [procfwk].[Orchestrators] d ON p.[OrchestratorId] = d.[OrchestratorId] WHERE p.[Enabled] = 1 AND s.[Enabled] = 1; SELECT @LocalExecutionId AS ExecutionId; END ELSE IF ([procfwk].[GetPropertyValueInternal]('UseExecutionBatches')) = '1' BEGIN DELETE FROM [procfwk].[CurrentExecution] WHERE [LocalExecutionId] = @LocalExecutionId; SELECT @BatchId = [BatchId] FROM [procfwk].[BatchExecution] WHERE [ExecutionId] = @LocalExecutionId; --defensive check IF NOT EXISTS ( SELECT 1 FROM [procfwk].[Pipelines] p INNER JOIN [procfwk].[Stages] s ON p.[StageId] = s.[StageId] INNER JOIN [procfwk].[Orchestrators] d ON p.[OrchestratorId] = d.[OrchestratorId] INNER JOIN [procfwk].[BatchStageLink] b ON b.[StageId] = s.[StageId] WHERE b.[BatchId] = @BatchId AND p.[Enabled] = 1 AND s.[Enabled] = 1 ) BEGIN RAISERROR('Requested execution run does not contain any enabled stages/pipelines.',16,1); RETURN 0; END; INSERT INTO [procfwk].[CurrentExecution] ( [LocalExecutionId], [StageId], [PipelineId], [CallingOrchestratorName], [ResourceGroupName], [OrchestratorType], [OrchestratorName], [PipelineName] ) SELECT @LocalExecutionId, p.[StageId], p.[PipelineId], @CallingOrchestratorName, d.[ResourceGroupName], d.[OrchestratorType], d.[OrchestratorName], p.[PipelineName] FROM [procfwk].[Pipelines] p INNER JOIN [procfwk].[Stages] s ON p.[StageId] = s.[StageId] INNER JOIN [procfwk].[Orchestrators] d ON p.[OrchestratorId] = d.[OrchestratorId] INNER JOIN [procfwk].[BatchStageLink] b ON b.[StageId] = s.[StageId] WHERE b.[BatchId] = @BatchId AND p.[Enabled] = 1 AND s.[Enabled] = 1; SELECT @LocalExecutionId AS ExecutionId; END; ALTER INDEX [IDX_GetPipelinesInStage] ON [procfwk].[CurrentExecution] REBUILD; END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/ExecutePrecursorProcedure.sql ================================================ CREATE PROCEDURE [procfwk].[ExecutePrecursorProcedure] AS BEGIN DECLARE @SQL VARCHAR(MAX) DECLARE @ErrorDetail NVARCHAR(MAX) IF OBJECT_ID([procfwk].[GetPropertyValueInternal]('ExecutionPrecursorProc')) IS NOT NULL BEGIN BEGIN TRY SET @SQL = [procfwk].[GetPropertyValueInternal]('ExecutionPrecursorProc'); EXEC(@SQL); END TRY BEGIN CATCH SELECT @ErrorDetail = 'Precursor procedure failed with error: ' + ERROR_MESSAGE(); RAISERROR(@ErrorDetail,16,1); END CATCH END; ELSE BEGIN PRINT 'Precursor object not found in database.'; END; END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/ExecutionWrapper.sql ================================================ CREATE PROCEDURE [procfwk].[ExecutionWrapper] ( @CallingOrchestratorName NVARCHAR(200) = NULL, @BatchName VARCHAR(255) = NULL ) AS BEGIN SET NOCOUNT ON; DECLARE @RestartStatus BIT DECLARE @BatchId UNIQUEIDENTIFIER DECLARE @BLocalExecutionId UNIQUEIDENTIFIER --declared here for batches IF @CallingOrchestratorName IS NULL SET @CallingOrchestratorName = 'Unknown'; --get restart overide property SELECT @RestartStatus = [procfwk].[GetPropertyValueInternal]('OverideRestart') IF([procfwk].[GetPropertyValueInternal]('UseExecutionBatches')) = '0' BEGIN SET @BatchId = NULL; --check for running execution IF EXISTS ( SELECT * FROM [procfwk].[CurrentExecution] WHERE ISNULL([PipelineStatus],'') = 'Running' ) BEGIN RAISERROR('There is already an execution run in progress. Stop this via the Orchestrator before restarting.',16,1); RETURN 0; END; --reset and restart execution IF EXISTS ( SELECT 1 FROM [procfwk].[CurrentExecution] WHERE ISNULL([PipelineStatus],'') <> 'Success' ) AND @RestartStatus = 0 BEGIN EXEC [procfwk].[ResetExecution] END --capture failed execution and run new anyway ELSE IF EXISTS ( SELECT 1 FROM [procfwk].[CurrentExecution] ) AND @RestartStatus = 1 BEGIN EXEC [procfwk].[UpdateExecutionLog] @PerformErrorCheck = 0; --Special case when OverideRestart = 1; EXEC [procfwk].[CreateNewExecution] @CallingOrchestratorName = @CallingOrchestratorName END --no restart considerations, just create new execution ELSE BEGIN EXEC [procfwk].[CreateNewExecution] @CallingOrchestratorName = @CallingOrchestratorName END END ELSE IF ([procfwk].[GetPropertyValueInternal]('UseExecutionBatches')) = '1' BEGIN IF @BatchName IS NULL BEGIN RAISERROR('A NULL batch name cannot be passed when the UseExecutionBatches property is set to 1 (true).',16,1); RETURN 0; END; SELECT @BatchId = [BatchId] FROM [procfwk].[Batches] WHERE [BatchName] = @BatchName; --create local execution id now for the batch EXEC [procfwk].[BatchWrapper] @BatchId = @BatchId, @LocalExecutionId = @BLocalExecutionId OUTPUT; --reset and restart execution IF EXISTS ( SELECT 1 FROM [procfwk].[CurrentExecution] WHERE [LocalExecutionId] = @BLocalExecutionId AND ISNULL([PipelineStatus],'') <> 'Success' ) AND @RestartStatus = 0 BEGIN EXEC [procfwk].[ResetExecution] @LocalExecutionId = @BLocalExecutionId; END --capture failed execution and run new anyway ELSE IF EXISTS ( SELECT 1 FROM [procfwk].[CurrentExecution] WHERE [LocalExecutionId] = @BLocalExecutionId ) AND @RestartStatus = 1 BEGIN EXEC [procfwk].[UpdateExecutionLog] @PerformErrorCheck = 0, --Special case when OverideRestart = 1; @ExecutionId = @BLocalExecutionId; EXEC [procfwk].[CreateNewExecution] @CallingOrchestratorName = @CallingOrchestratorName, @LocalExecutionId = @BLocalExecutionId; END --no restart considerations, just create new execution ELSE BEGIN EXEC [procfwk].[CreateNewExecution] @CallingOrchestratorName = @CallingOrchestratorName, @LocalExecutionId = @BLocalExecutionId; END END ELSE BEGIN --metadata integrity checks should mean this condition is never hit RAISERROR('Unknown batch handling configuration. Update properties with UseExecutionBatches value and try again.',16,1); RETURN 0; END END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/GetEmailAlertParts.sql ================================================ CREATE PROCEDURE [procfwk].[GetEmailAlertParts] ( @PipelineId INT ) AS BEGIN SET NOCOUNT ON; DECLARE @ToRecipients NVARCHAR(MAX) = '' DECLARE @CcRecipients NVARCHAR(MAX) = '' DECLARE @BccRecipients NVARCHAR(MAX) = '' DECLARE @EmailSubject NVARCHAR(500) DECLARE @EmailBody NVARCHAR(MAX) DECLARE @EmailImportance VARCHAR(5) DECLARE @OutcomeBitValue INT --map pipeline status to alert outcome bit value SELECT @OutcomeBitValue = ao.[BitValue] FROM [procfwk].[CurrentExecution] ce INNER JOIN [procfwk].[AlertOutcomes] ao ON ce.[PipelineStatus] = ao.[PipelineOutcomeStatus] WHERE ce.[PipelineId] = @PipelineId; --get to recipients SELECT @ToRecipients += r.[EmailAddress] + ',' FROM [procfwk].[PipelineAlertLink] al INNER JOIN [procfwk].[Recipients] r ON al.[RecipientId] = r.[RecipientId] WHERE al.[PipelineId] = @PipelineId AND al.[Enabled] = 1 AND r.[Enabled] = 1 AND UPPER(r.[MessagePreference]) = 'TO' AND ( al.[OutcomesBitValue] & @OutcomeBitValue <> 0 OR al.[OutcomesBitValue] & 1 <> 0 --all ); IF (@ToRecipients <> '') SET @ToRecipients = LEFT(@ToRecipients,LEN(@ToRecipients)-1); --get cc recipients SELECT @CcRecipients += r.[EmailAddress] + ',' FROM [procfwk].[PipelineAlertLink] al INNER JOIN [procfwk].[Recipients] r ON al.[RecipientId] = r.[RecipientId] WHERE al.[PipelineId] = @PipelineId AND al.[Enabled] = 1 AND r.[Enabled] = 1 AND UPPER(r.[MessagePreference]) = 'CC' AND ( al.[OutcomesBitValue] & @OutcomeBitValue <> 0 OR al.[OutcomesBitValue] & 1 <> 0 --all ); IF (@CcRecipients <> '') SET @CcRecipients = LEFT(@CcRecipients,LEN(@CcRecipients)-1); --get bcc recipients SELECT @BccRecipients += r.[EmailAddress] + ',' FROM [procfwk].[PipelineAlertLink] al INNER JOIN [procfwk].[Recipients] r ON al.[RecipientId] = r.[RecipientId] WHERE al.[PipelineId] = @PipelineId AND al.[Enabled] = 1 AND r.[Enabled] = 1 AND UPPER(r.[MessagePreference]) = 'BCC' AND ( al.[OutcomesBitValue] & @OutcomeBitValue <> 0 OR al.[OutcomesBitValue] & 1 <> 0 --all ); IF (@BccRecipients <> '') SET @BccRecipients = LEFT(@BccRecipients,LEN(@BccRecipients)-1); --get email template SELECT @EmailBody = [PropertyValue] FROM [procfwk].[CurrentProperties] WHERE [PropertyName] = 'EmailAlertBodyTemplate'; --set subject, body and importance SELECT TOP (1) --subject @EmailSubject = 'ProcFwk Alert: ' + [PipelineName] + ' - ' + [PipelineStatus], --body @EmailBody = REPLACE(@EmailBody,'##PipelineName###',[PipelineName]), @EmailBody = REPLACE(@EmailBody,'##Status###',[PipelineStatus]), @EmailBody = REPLACE(@EmailBody,'##ExecId###',CAST([LocalExecutionId] AS VARCHAR(36))), @EmailBody = REPLACE(@EmailBody,'##RunId###',CAST([PipelineRunId] AS VARCHAR(36))), @EmailBody = REPLACE(@EmailBody,'##StartDateTime###',CONVERT(VARCHAR(30), [StartDateTime], 120)), @EmailBody = CASE WHEN [EndDateTime] IS NULL THEN REPLACE(@EmailBody,'##EndDateTime###','N/A') ELSE REPLACE(@EmailBody,'##EndDateTime###',CONVERT(VARCHAR(30), [EndDateTime], 120)) END, @EmailBody = CASE WHEN [EndDateTime] IS NULL THEN REPLACE(@EmailBody,'##Duration###','N/A') ELSE REPLACE(@EmailBody,'##Duration###',CAST(DATEDIFF(MINUTE, [StartDateTime], [EndDateTime]) AS VARCHAR(30))) END, @EmailBody = REPLACE(@EmailBody,'##CalledByOrc###',[CallingOrchestratorName]), @EmailBody = REPLACE(@EmailBody,'##ExecutedByOrcType###',[OrchestratorType]), @EmailBody = REPLACE(@EmailBody,'##ExecutedByOrc###',[OrchestratorName]), --importance @EmailImportance = CASE [PipelineStatus] WHEN 'Success' THEN 'Low' WHEN 'Failed' THEN 'High' ELSE 'Normal' END FROM [procfwk].[CurrentExecution] WHERE [PipelineId] = @PipelineId ORDER BY [StartDateTime] DESC; --precaution IF @EmailBody IS NULL SET @EmailBody = 'Internal error. Failed to create profwk email alert body. Execute procedure [procfwk].[GetEmailAlertParts] with pipeline Id: ' + CAST(@PipelineId AS VARCHAR(30)) + ' to debug.'; --return email parts SELECT @ToRecipients AS emailRecipients, @CcRecipients AS emailCcRecipients, @BccRecipients AS emailBccRecipients, @EmailSubject AS emailSubject, @EmailBody AS emailBody, @EmailImportance AS emailImportance; END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/GetFrameworkOrchestratorDetails.sql ================================================ CREATE PROCEDURE [procfwk].[GetFrameworkOrchestratorDetails] ( @CallingOrchestratorName NVARCHAR(200) ) AS BEGIN SET NOCOUNT ON; DECLARE @FrameworkOrchestrator NVARCHAR(200) --defensive check SELECT @FrameworkOrchestrator = UPPER([OrchestratorName]), @CallingOrchestratorName = UPPER(@CallingOrchestratorName) FROM [procfwk].[Orchestrators] WHERE [IsFrameworkOrchestrator] = 1; IF(@FrameworkOrchestrator <> @CallingOrchestratorName) BEGIN RAISERROR('Orchestrator mismatch. Calling orchestrator does not match expected IsFrameworkOrchestrator name.',16,1); RETURN 0; END --orchestrator detials SELECT [SubscriptionId], [ResourceGroupName], [OrchestratorName], [OrchestratorType] FROM [procfwk].[Orchestrators] WHERE [IsFrameworkOrchestrator] = 1; END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/GetPipelineParameters.sql ================================================ CREATE PROCEDURE [procfwk].[GetPipelineParameters] ( @PipelineId INT ) AS BEGIN SET NOCOUNT ON; DECLARE @Json VARCHAR(MAX) = '' --get parameters if required for worker pipeline IF NOT EXISTS ( SELECT [ParameterId] FROM [procfwk].[PipelineParameters] WHERE [PipelineId] = @PipelineId ) BEGIN SET @Json = '' --Can't return NULL. Would break ADF expression. END ELSE BEGIN SELECT @Json += CASE WHEN [ParameterValue] IS NULL THEN '' --don't add pair so ADF uses default ELSE '"' + [ParameterName] + '": "' + STRING_ESCAPE([ParameterValue],'json') + '",' END FROM [procfwk].[PipelineParameters] WHERE [PipelineId] = @PipelineId; --handle parameter(s) with a NULL values IF LEN(@Json) > 0 BEGIN --JSON snippet gets injected into Azure Function body request via Orchestrator expressions. --Comma used to support Orchestrator expression. SET @Json = ',"pipelineParameters": {' + LEFT(@Json,LEN(@Json)-1) + '}' --update current execution log if this is a runtime request UPDATE [procfwk].[CurrentExecution] SET --add extra braces to make JSON string valid in logs [PipelineParamsUsed] = '{ ' + RIGHT(@Json,LEN(@Json)-1) + ' }' WHERE [PipelineId] = @PipelineId; --set last values values UPDATE [procfwk].[PipelineParameters] SET [ParameterValueLastUsed] = [ParameterValue] WHERE [PipelineId] = @PipelineId; END; END; --return JSON snippet SELECT @Json AS Params END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/GetPipelinesInStage.sql ================================================ CREATE PROCEDURE [procfwk].[GetPipelinesInStage] ( @ExecutionId UNIQUEIDENTIFIER, @StageId INT ) AS BEGIN SET NOCOUNT ON; SELECT [PipelineId] FROM [procfwk].[CurrentExecution] WHERE [LocalExecutionId] = @ExecutionId AND [StageId] = @StageId AND ISNULL([PipelineStatus],'') <> 'Success' AND [IsBlocked] <> 1 ORDER BY [PipelineId] ASC; END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/GetPropertyValue.sql ================================================ CREATE PROCEDURE [procfwk].[GetPropertyValue] ( @PropertyName VARCHAR(128) ) AS BEGIN DECLARE @ErrorDetail NVARCHAR(4000) = '' --defensive checks IF NOT EXISTS ( SELECT * FROM [procfwk].[Properties] WHERE [PropertyName] = @PropertyName ) BEGIN SET @ErrorDetail = 'Invalid property name provided. Property does not exist.' RAISERROR(@ErrorDetail, 16, 1); RETURN 0; END ELSE IF NOT EXISTS ( SELECT * FROM [procfwk].[Properties] WHERE [PropertyName] = @PropertyName AND [ValidTo] IS NULL ) BEGIN SET @ErrorDetail = 'Property name provided does not have a current valid version of the required value.' RAISERROR(@ErrorDetail, 16, 1); RETURN 0; END --get valid property value ELSE BEGIN SELECT [PropertyValue] FROM [procfwk].[CurrentProperties] WHERE [PropertyName] = @PropertyName END END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/GetStages.sql ================================================ CREATE PROCEDURE [procfwk].[GetStages] ( @ExecutionId UNIQUEIDENTIFIER ) AS BEGIN SET NOCOUNT ON; --defensive check IF NOT EXISTS ( SELECT 1 FROM [procfwk].[CurrentExecution] WHERE [LocalExecutionId] = @ExecutionId AND ISNULL([PipelineStatus],'') <> 'Success' ) BEGIN RAISERROR('Requested execution run does not contain any enabled stages/pipelines.',16,1); RETURN 0; END; SELECT DISTINCT [StageId] FROM [procfwk].[CurrentExecution] WHERE [LocalExecutionId] = @ExecutionId AND ISNULL([PipelineStatus],'') <> 'Success' ORDER BY [StageId] ASC END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/GetWorkerAuthDetails.sql ================================================ CREATE PROCEDURE [procfwk].[GetWorkerAuthDetails] ( @ExecutionId UNIQUEIDENTIFIER, @StageId INT, @PipelineId INT ) AS BEGIN SET NOCOUNT ON; DECLARE @TenId NVARCHAR(MAX) DECLARE @SubId NVARCHAR(MAX) DECLARE @AppId NVARCHAR(MAX) DECLARE @AppSecret NVARCHAR(MAX) DECLARE @OrchestratorName NVARCHAR(200) DECLARE @OrchestratorType CHAR(3) DECLARE @PipelineName NVARCHAR(200) SELECT @PipelineName = [PipelineName], @OrchestratorName = [OrchestratorName], @OrchestratorType = [OrchestratorType] FROM [procfwk].[CurrentExecution] WHERE [LocalExecutionId] = @ExecutionId AND [StageId] = @StageId AND [PipelineId] = @PipelineId; IF ([procfwk].[GetPropertyValueInternal]('SPNHandlingMethod')) = 'StoreInDatabase' BEGIN --get auth details regardless of being pipeline specific and regardless of a pipeline param being passed ;WITH cte AS ( SELECT DISTINCT Sub.[TenantId], Sub.[SubscriptionId], S.[PrincipalId] AS AppId, CAST(DECRYPTBYPASSPHRASE(CONCAT(@OrchestratorName, @OrchestratorType, @PipelineName), S.[PrincipalSecret]) AS NVARCHAR(MAX)) AS AppSecret FROM [dbo].[ServicePrincipals] S INNER JOIN [procfwk].[PipelineAuthLink] L ON S.[CredentialId] = L.[CredentialId] INNER JOIN [procfwk].[Pipelines] P ON L.[PipelineId] = P.[PipelineId] INNER JOIN [procfwk].[Orchestrators] D ON P.[OrchestratorId] = D.[OrchestratorId] AND L.[OrchestratorId] = D.[OrchestratorId] INNER JOIN [procfwk].[Subscriptions] Sub ON D.[SubscriptionId] = Sub.[SubscriptionId] WHERE P.[PipelineName] = @PipelineName AND D.[OrchestratorName] = @OrchestratorName AND D.[OrchestratorType] = @OrchestratorType UNION SELECT DISTINCT Sub.[TenantId], Sub.[SubscriptionId], S.[PrincipalId] AS AppId, CAST(DECRYPTBYPASSPHRASE(CONCAT(@OrchestratorName, @OrchestratorType), S.[PrincipalSecret]) AS NVARCHAR(MAX)) AS AppSecret FROM [dbo].[ServicePrincipals] S INNER JOIN [procfwk].[PipelineAuthLink] L ON S.[CredentialId] = L.[CredentialId] INNER JOIN [procfwk].[Orchestrators] D ON L.[OrchestratorId] = D.[OrchestratorId] INNER JOIN [procfwk].[Subscriptions] Sub ON D.[SubscriptionId] = Sub.[SubscriptionId] WHERE D.[OrchestratorName] = @OrchestratorName AND D.[OrchestratorType] = @OrchestratorType ) SELECT TOP 1 @TenId = [TenantId], @SubId = [SubscriptionId], @AppId = [AppId], @AppSecret = [AppSecret] FROM cte WHERE [AppSecret] IS NOT NULL END ELSE IF ([procfwk].[GetPropertyValueInternal]('SPNHandlingMethod')) = 'StoreInKeyVault' BEGIN --get auth details regardless of being pipeline specific and regardless of a pipeline param being passed ;WITH cte AS ( SELECT DISTINCT Sub.[TenantId], Sub.[SubscriptionId], S.[PrincipalIdUrl] AS AppId, S.[PrincipalSecretUrl] AS AppSecret FROM [dbo].[ServicePrincipals] S INNER JOIN [procfwk].[PipelineAuthLink] L ON S.[CredentialId] = L.[CredentialId] INNER JOIN [procfwk].[Pipelines] P ON L.[PipelineId] = P.[PipelineId] INNER JOIN [procfwk].[Orchestrators] D ON P.[OrchestratorId] = D.[OrchestratorId] AND L.[OrchestratorId] = D.[OrchestratorId] INNER JOIN [procfwk].[Subscriptions] Sub ON D.[SubscriptionId] = Sub.[SubscriptionId] WHERE P.[PipelineName] = @PipelineName AND D.[OrchestratorName] = @OrchestratorName AND D.[OrchestratorType] = @OrchestratorType UNION SELECT DISTINCT Sub.[TenantId], Sub.[SubscriptionId], S.[PrincipalIdUrl] AS AppId, S.[PrincipalSecretUrl] AS AppSecret FROM [dbo].[ServicePrincipals] S INNER JOIN [procfwk].[PipelineAuthLink] L ON S.[CredentialId] = L.[CredentialId] INNER JOIN [procfwk].[Orchestrators] D ON L.[OrchestratorId] = D.[OrchestratorId] INNER JOIN [procfwk].[Subscriptions] Sub ON D.[SubscriptionId] = Sub.[SubscriptionId] WHERE D.[OrchestratorName] = @OrchestratorName AND D.[OrchestratorType] = @OrchestratorType ) SELECT TOP 1 @TenId = [TenantId], @SubId = [SubscriptionId], @AppId = [AppId], @AppSecret = [AppSecret] FROM cte WHERE [AppSecret] IS NOT NULL END ELSE BEGIN RAISERROR('Unknown SPN retrieval method.',16,1); RETURN 0; END --return usable values SELECT @TenId AS TenantId, @SubId AS SubscriptionId, @AppId AS AppId, @AppSecret AS AppSecret END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/GetWorkerDetailsWrapper.sql ================================================ CREATE PROCEDURE [procfwk].[GetWorkerDetailsWrapper] ( @ExecutionId UNIQUEIDENTIFIER, @StageId INT, @PipelineId INT ) AS BEGIN /* Created this proc just to reduce and refactor the number of pipeline activity calls needed due to the Microsoft enforced limit of 40 activities per pipeline. */ SET NOCOUNT ON; DECLARE @WorkerAuthDetails TABLE ( [tenantId] UNIQUEIDENTIFIER NULL, [applicationId] NVARCHAR(MAX) NULL, [authenticationKey] NVARCHAR(MAX) NULL, [subscriptionId] UNIQUEIDENTIFIER NULL ) DECLARE @WorkerDetails TABLE ( [resourceGroupName] NVARCHAR(200) NULL, [orchestratorName] NVARCHAR(200) NULL, [orchestratorType] CHAR(3) NULL, [pipelineName] NVARCHAR(200) NULL ) --get work auth details INSERT INTO @WorkerAuthDetails ( [tenantId], [subscriptionId], [applicationId], [authenticationKey] ) EXEC [procfwk].[GetWorkerAuthDetails] @ExecutionId = @ExecutionId, @StageId = @StageId, @PipelineId = @PipelineId; --get main worker details INSERT INTO @WorkerDetails ( [pipelineName], [orchestratorName], [orchestratorType], [resourceGroupName] ) EXEC [procfwk].[GetWorkerPipelineDetails] @ExecutionId = @ExecutionId, @StageId = @StageId, @PipelineId = @PipelineId; --return all details SELECT ad.[tenantId], ad.[applicationId], ad.[authenticationKey], ad.[subscriptionId], d.[resourceGroupName], d.[orchestratorName], d.[orchestratorType], d.[pipelineName] FROM @WorkerDetails d CROSS JOIN @WorkerAuthDetails ad; END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/GetWorkerPipelineDetails.sql ================================================ CREATE PROCEDURE [procfwk].[GetWorkerPipelineDetails] ( @ExecutionId UNIQUEIDENTIFIER, @StageId INT, @PipelineId INT ) AS BEGIN SET NOCOUNT ON; SELECT [PipelineName], [OrchestratorName], [OrchestratorType], [ResourceGroupName] FROM [procfwk].[CurrentExecution] WHERE [LocalExecutionId] = @ExecutionId AND [StageId] = @StageId AND [PipelineId] = @PipelineId; END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/ResetExecution.sql ================================================ CREATE PROCEDURE [procfwk].[ResetExecution] ( @LocalExecutionId UNIQUEIDENTIFIER = NULL ) AS BEGIN SET NOCOUNT ON; IF([procfwk].[GetPropertyValueInternal]('UseExecutionBatches')) = '0' BEGIN --capture any pipelines that might be in an unexpected state INSERT INTO [procfwk].[ExecutionLog] ( [LocalExecutionId], [StageId], [PipelineId], [CallingOrchestratorName], [ResourceGroupName], [OrchestratorType], [OrchestratorName], [PipelineName], [StartDateTime], [PipelineStatus], [EndDateTime] ) SELECT [LocalExecutionId], [StageId], [PipelineId], [CallingOrchestratorName], [ResourceGroupName], [OrchestratorType], [OrchestratorName], [PipelineName], [StartDateTime], 'Unknown', [EndDateTime] FROM [procfwk].[CurrentExecution] WHERE --these are predicted states [PipelineStatus] NOT IN ( 'Success', 'Failed', 'Blocked', 'Cancelled' ); --reset status ready for next attempt UPDATE [procfwk].[CurrentExecution] SET [StartDateTime] = NULL, [EndDateTime] = NULL, [PipelineStatus] = NULL, [LastStatusCheckDateTime] = NULL, [PipelineRunId] = NULL, [PipelineParamsUsed] = NULL, [IsBlocked] = 0 WHERE ISNULL([PipelineStatus],'') <> 'Success' OR [IsBlocked] = 1; --return current execution id SELECT DISTINCT [LocalExecutionId] AS ExecutionId FROM [procfwk].[CurrentExecution]; END ELSE IF ([procfwk].[GetPropertyValueInternal]('UseExecutionBatches')) = '1' BEGIN --capture any pipelines that might be in an unexpected state INSERT INTO [procfwk].[ExecutionLog] ( [LocalExecutionId], [StageId], [PipelineId], [CallingOrchestratorName], [ResourceGroupName], [OrchestratorType], [OrchestratorName], [PipelineName], [StartDateTime], [PipelineStatus], [EndDateTime] ) SELECT [LocalExecutionId], [StageId], [PipelineId], [CallingOrchestratorName], [ResourceGroupName], [OrchestratorType], [OrchestratorName], [PipelineName], [StartDateTime], 'Unknown', [EndDateTime] FROM [procfwk].[CurrentExecution] WHERE [LocalExecutionId] = @LocalExecutionId --these are predicted states AND [PipelineStatus] NOT IN ( 'Success', 'Failed', 'Blocked', 'Cancelled' ); --reset status ready for next attempt UPDATE [procfwk].[CurrentExecution] SET [StartDateTime] = NULL, [EndDateTime] = NULL, [PipelineStatus] = NULL, [LastStatusCheckDateTime] = NULL, [PipelineRunId] = NULL, [PipelineParamsUsed] = NULL, [IsBlocked] = 0 WHERE [LocalExecutionId] = @LocalExecutionId AND ISNULL([PipelineStatus],'') <> 'Success' OR [IsBlocked] = 1; UPDATE [procfwk].[BatchExecution] SET [EndDateTime] = NULL, [BatchStatus] = 'Running' WHERE [ExecutionId] = @LocalExecutionId; SELECT @LocalExecutionId AS ExecutionId END; END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/SetErrorLogDetails.sql ================================================ CREATE PROCEDURE [procfwk].[SetErrorLogDetails] ( @LocalExecutionId UNIQUEIDENTIFIER, @JsonErrorDetails VARCHAR(MAX) ) AS BEGIN SET NOCOUNT ON; INSERT INTO [procfwk].[ErrorLog] ( [LocalExecutionId], [PipelineRunId], [ActivityRunId], [ActivityName], [ActivityType], [ErrorCode], [ErrorType], [ErrorMessage] ) SELECT @LocalExecutionId, Base.[RunId], ErrorDetail.[ActivityRunId], ErrorDetail.[ActivityName], ErrorDetail.[ActivityType], ErrorDetail.[ErrorCode], ErrorDetail.[ErrorType], ErrorDetail.[ErrorMessage] FROM OPENJSON(@JsonErrorDetails) WITH ( [RunId] UNIQUEIDENTIFIER, [Errors] NVARCHAR(MAX) AS JSON ) AS Base CROSS APPLY OPENJSON (Base.[Errors]) WITH ( [ActivityRunId] UNIQUEIDENTIFIER, [ActivityName] VARCHAR(100), [ActivityType] VARCHAR(100), [ErrorCode] VARCHAR(100), [ErrorType] VARCHAR(100), [ErrorMessage] VARCHAR(MAX) ) AS ErrorDetail END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/SetExecutionBlockDependants.sql ================================================ CREATE PROCEDURE [procfwk].[SetExecutionBlockDependants] ( @ExecutionId UNIQUEIDENTIFIER = NULL, @PipelineId INT ) AS BEGIN --update dependents status UPDATE ce SET ce.[PipelineStatus] = 'Blocked', ce.[IsBlocked] = 1 FROM [procfwk].[PipelineDependencies] pe INNER JOIN [procfwk].[CurrentExecution] ce ON pe.[DependantPipelineId] = ce.[PipelineId] WHERE ce.[LocalExecutionId] = @ExecutionId AND pe.[PipelineId] = @PipelineId END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/SetLogActivityFailed.sql ================================================ CREATE PROCEDURE [procfwk].[SetLogActivityFailed] ( @ExecutionId UNIQUEIDENTIFIER, @StageId INT, @PipelineId INT, @CallingActivity VARCHAR(255) ) AS BEGIN SET NOCOUNT ON; --mark specific failure pipeline UPDATE [procfwk].[CurrentExecution] SET [PipelineStatus] = @CallingActivity + 'Error' WHERE [LocalExecutionId] = @ExecutionId AND [StageId] = @StageId AND [PipelineId] = @PipelineId --persist failed pipeline records to long term log INSERT INTO [procfwk].[ExecutionLog] ( [LocalExecutionId], [StageId], [PipelineId], [CallingOrchestratorName], [ResourceGroupName], [OrchestratorType], [OrchestratorName], [PipelineName], [StartDateTime], [PipelineStatus], [EndDateTime], [PipelineRunId], [PipelineParamsUsed] ) SELECT [LocalExecutionId], [StageId], [PipelineId], [CallingOrchestratorName], [ResourceGroupName], [OrchestratorType], [OrchestratorName], [PipelineName], [StartDateTime], [PipelineStatus], [EndDateTime], [PipelineRunId], [PipelineParamsUsed] FROM [procfwk].[CurrentExecution] WHERE [LocalExecutionId] = @ExecutionId AND [PipelineStatus] = @CallingActivity + 'Error' AND [StageId] = @StageId AND [PipelineId] = @PipelineId --decide how to proceed with error/failure depending on framework property configuration IF ([procfwk].[GetPropertyValueInternal]('FailureHandling')) = 'None' BEGIN --do nothing allow processing to carry on regardless RETURN 0; END; ELSE IF ([procfwk].[GetPropertyValueInternal]('FailureHandling')) = 'Simple' BEGIN --flag all downstream stages as blocked UPDATE [procfwk].[CurrentExecution] SET [PipelineStatus] = 'Blocked', [IsBlocked] = 1 WHERE [LocalExecutionId] = @ExecutionId AND [StageId] > @StageId; --update batch if applicable IF ([procfwk].[GetPropertyValueInternal]('UseExecutionBatches')) = '1' BEGIN UPDATE [procfwk].[BatchExecution] SET [BatchStatus] = 'Stopping' --special case when its an activity failure to call stop ready for restart WHERE [ExecutionId] = @ExecutionId AND [BatchStatus] = 'Running'; END; END; ELSE IF ([procfwk].[GetPropertyValueInternal]('FailureHandling')) = 'DependencyChain' BEGIN EXEC [procfwk].[SetExecutionBlockDependants] @ExecutionId = @ExecutionId, @PipelineId = @PipelineId END; ELSE BEGIN RAISERROR('Unknown failure handling state.',16,1); RETURN 0; END; END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/SetLogPipelineCancelled.sql ================================================ CREATE PROCEDURE [procfwk].[SetLogPipelineCancelled] ( @ExecutionId UNIQUEIDENTIFIER, @StageId INT, @PipelineId INT, @CleanUpRun BIT = 0 ) AS BEGIN SET NOCOUNT ON; DECLARE @ErrorDetail VARCHAR(500); --mark specific failure pipeline UPDATE [procfwk].[CurrentExecution] SET [PipelineStatus] = 'Cancelled' WHERE [LocalExecutionId] = @ExecutionId AND [StageId] = @StageId AND [PipelineId] = @PipelineId --no need to block and log if done during a clean up cycle IF @CleanUpRun = 1 RETURN 0; --persist cancelled pipeline records to long term log INSERT INTO [procfwk].[ExecutionLog] ( [LocalExecutionId], [StageId], [PipelineId], [CallingOrchestratorName], [ResourceGroupName], [OrchestratorType], [OrchestratorName], [PipelineName], [StartDateTime], [PipelineStatus], [EndDateTime], [PipelineRunId], [PipelineParamsUsed] ) SELECT [LocalExecutionId], [StageId], [PipelineId], [CallingOrchestratorName], [ResourceGroupName], [OrchestratorType], [OrchestratorName], [PipelineName], [StartDateTime], [PipelineStatus], [EndDateTime], [PipelineRunId], [PipelineParamsUsed] FROM [procfwk].[CurrentExecution] WHERE [LocalExecutionId] = @ExecutionId AND [PipelineStatus] = 'Cancelled' AND [StageId] = @StageId AND [PipelineId] = @PipelineId; --block down stream stages? IF ([procfwk].[GetPropertyValueInternal]('CancelledWorkerResultBlocks')) = 1 BEGIN --decide how to proceed with error/failure depending on framework property configuration IF ([procfwk].[GetPropertyValueInternal]('FailureHandling')) = 'None' BEGIN --do nothing allow processing to carry on regardless RETURN 0; END; ELSE IF ([procfwk].[GetPropertyValueInternal]('FailureHandling')) = 'Simple' BEGIN --flag all downstream stages as blocked UPDATE [procfwk].[CurrentExecution] SET [PipelineStatus] = 'Blocked', [IsBlocked] = 1 WHERE [LocalExecutionId] = @ExecutionId AND [StageId] > @StageId --update batch if applicable IF ([procfwk].[GetPropertyValueInternal]('UseExecutionBatches')) = '1' BEGIN UPDATE [procfwk].[BatchExecution] SET [BatchStatus] = 'Stopping' WHERE [ExecutionId] = @ExecutionId AND [BatchStatus] = 'Running'; END; SET @ErrorDetail = 'Pipeline execution has a cancelled status. Blocking downstream stages as a precaution.' RAISERROR(@ErrorDetail,16,1); RETURN 0; END; ELSE IF ([procfwk].[GetPropertyValueInternal]('FailureHandling')) = 'DependencyChain' BEGIN EXEC [procfwk].[SetExecutionBlockDependants] @ExecutionId = @ExecutionId, @PipelineId = @PipelineId END; ELSE BEGIN RAISERROR('Cancelled execution failure handling state.',16,1); RETURN 0; END; END; END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/SetLogPipelineChecking.sql ================================================ CREATE PROCEDURE [procfwk].[SetLogPipelineChecking] ( @ExecutionId UNIQUEIDENTIFIER, @StageId INT, @PipelineId INT ) AS BEGIN SET NOCOUNT ON; UPDATE [procfwk].[CurrentExecution] SET [PipelineStatus] = 'Checking' WHERE [LocalExecutionId] = @ExecutionId AND [StageId] = @StageId AND [PipelineId] = @PipelineId END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/SetLogPipelineFailed.sql ================================================ CREATE PROCEDURE [procfwk].[SetLogPipelineFailed] ( @ExecutionId UNIQUEIDENTIFIER, @StageId INT, @PipelineId INT, @RunId UNIQUEIDENTIFIER = NULL ) AS BEGIN SET NOCOUNT ON; DECLARE @ErrorDetail VARCHAR(500) --mark specific failure pipeline UPDATE [procfwk].[CurrentExecution] SET [PipelineStatus] = 'Failed' WHERE [LocalExecutionId] = @ExecutionId AND [StageId] = @StageId AND [PipelineId] = @PipelineId --persist failed pipeline records to long term log INSERT INTO [procfwk].[ExecutionLog] ( [LocalExecutionId], [StageId], [PipelineId], [CallingOrchestratorName], [ResourceGroupName], [OrchestratorType], [OrchestratorName], [PipelineName], [StartDateTime], [PipelineStatus], [EndDateTime], [PipelineRunId], [PipelineParamsUsed] ) SELECT [LocalExecutionId], [StageId], [PipelineId], [CallingOrchestratorName], [ResourceGroupName], [OrchestratorType], [OrchestratorName], [PipelineName], [StartDateTime], [PipelineStatus], [EndDateTime], [PipelineRunId], [PipelineParamsUsed] FROM [procfwk].[CurrentExecution] WHERE [LocalExecutionId] = @ExecutionId AND [PipelineStatus] = 'Failed' AND [StageId] = @StageId AND [PipelineId] = @PipelineId; IF ([procfwk].[GetPropertyValueInternal]('FailureHandling')) = 'None' BEGIN --do nothing allow processing to carry on regardless RETURN 0; END; ELSE IF ([procfwk].[GetPropertyValueInternal]('FailureHandling')) = 'Simple' BEGIN --flag all downstream stages as blocked UPDATE [procfwk].[CurrentExecution] SET [PipelineStatus] = 'Blocked', [IsBlocked] = 1 WHERE [LocalExecutionId] = @ExecutionId AND [StageId] > @StageId --raise error to stop processing IF @RunId IS NOT NULL BEGIN SET @ErrorDetail = 'Pipeline execution failed. Check Run ID: ' + CAST(@RunId AS CHAR(36)) + ' in ADF monitoring for details.' END; ELSE BEGIN SET @ErrorDetail = 'Pipeline execution failed. See ADF monitoring for details.' END; --update batch if applicable IF ([procfwk].[GetPropertyValueInternal]('UseExecutionBatches')) = '1' BEGIN UPDATE [procfwk].[BatchExecution] SET [BatchStatus] = 'Stopping' WHERE [ExecutionId] = @ExecutionId AND [BatchStatus] = 'Running'; END; RAISERROR(@ErrorDetail,16,1); RETURN 0; END; ELSE IF ([procfwk].[GetPropertyValueInternal]('FailureHandling')) = 'DependencyChain' BEGIN EXEC [procfwk].[SetExecutionBlockDependants] @ExecutionId = @ExecutionId, @PipelineId = @PipelineId END; ELSE BEGIN RAISERROR('Unknown failure handling state.',16,1); RETURN 0; END; END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/SetLogPipelineLastStatusCheck.sql ================================================ CREATE PROCEDURE [procfwk].[SetLogPipelineLastStatusCheck] ( @ExecutionId UNIQUEIDENTIFIER, @StageId INT, @PipelineId INT ) AS BEGIN SET NOCOUNT ON; UPDATE [procfwk].[CurrentExecution] SET [LastStatusCheckDateTime] = GETUTCDATE() WHERE [LocalExecutionId] = @ExecutionId AND [StageId] = @StageId AND [PipelineId] = @PipelineId END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/SetLogPipelineRunId.sql ================================================ CREATE PROCEDURE [procfwk].[SetLogPipelineRunId] ( @ExecutionId UNIQUEIDENTIFIER, @StageId INT, @PipelineId INT, @RunId UNIQUEIDENTIFIER = NULL ) AS BEGIN SET NOCOUNT ON; UPDATE [procfwk].[CurrentExecution] SET [PipelineRunId] = LOWER(@RunId) WHERE [LocalExecutionId] = @ExecutionId AND [StageId] = @StageId AND [PipelineId] = @PipelineId END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/SetLogPipelineRunning.sql ================================================ CREATE PROCEDURE procfwk.SetLogPipelineRunning ( @ExecutionId UNIQUEIDENTIFIER, @StageId INT, @PipelineId INT ) AS BEGIN SET NOCOUNT ON; UPDATE [procfwk].[CurrentExecution] SET --case for clean up runs [StartDateTime] = CASE WHEN [StartDateTime] IS NULL THEN GETUTCDATE() ELSE [StartDateTime] END, [PipelineStatus] = 'Running' WHERE [LocalExecutionId] = @ExecutionId AND [StageId] = @StageId AND [PipelineId] = @PipelineId END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/SetLogPipelineSuccess.sql ================================================ CREATE PROCEDURE procfwk.SetLogPipelineSuccess ( @ExecutionId UNIQUEIDENTIFIER, @StageId INT, @PipelineId INT ) AS BEGIN SET NOCOUNT ON; UPDATE [procfwk].[CurrentExecution] SET --case for clean up runs [EndDateTime] = CASE WHEN [EndDateTime] IS NULL THEN GETUTCDATE() ELSE [EndDateTime] END, [PipelineStatus] = 'Success' WHERE [LocalExecutionId] = @ExecutionId AND [StageId] = @StageId AND [PipelineId] = @PipelineId END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/SetLogPipelineUnknown.sql ================================================ CREATE PROCEDURE [procfwk].[SetLogPipelineUnknown] ( @ExecutionId UNIQUEIDENTIFIER, @StageId INT, @PipelineId INT, @CleanUpRun BIT = 0 ) AS BEGIN SET NOCOUNT ON; DECLARE @ErrorDetail VARCHAR(500); --mark specific failure pipeline UPDATE [procfwk].[CurrentExecution] SET [PipelineStatus] = 'Unknown' WHERE [LocalExecutionId] = @ExecutionId AND [StageId] = @StageId AND [PipelineId] = @PipelineId --no need to block and log if done during a clean up cycle IF @CleanUpRun = 1 RETURN 0; --persist unknown pipeline records to long term log INSERT INTO [procfwk].[ExecutionLog] ( [LocalExecutionId], [StageId], [PipelineId], [CallingOrchestratorName], [ResourceGroupName], [OrchestratorType], [OrchestratorName], [PipelineName], [StartDateTime], [PipelineStatus], [EndDateTime], [PipelineRunId], [PipelineParamsUsed] ) SELECT [LocalExecutionId], [StageId], [PipelineId], [CallingOrchestratorName], [ResourceGroupName], [OrchestratorType], [OrchestratorName], [PipelineName], [StartDateTime], [PipelineStatus], [EndDateTime], [PipelineRunId], [PipelineParamsUsed] FROM [procfwk].[CurrentExecution] WHERE [PipelineStatus] = 'Unknown' AND [StageId] = @StageId AND [PipelineId] = @PipelineId; --block down stream stages? IF ([procfwk].[GetPropertyValueInternal]('UnknownWorkerResultBlocks')) = 1 BEGIN --decide how to proceed with error/failure depending on framework property configuration IF ([procfwk].[GetPropertyValueInternal]('FailureHandling')) = 'None' BEGIN --do nothing allow processing to carry on regardless RETURN 0; END; ELSE IF ([procfwk].[GetPropertyValueInternal]('FailureHandling')) = 'Simple' BEGIN --flag all downstream stages as blocked UPDATE [procfwk].[CurrentExecution] SET [PipelineStatus] = 'Blocked', [IsBlocked] = 1 WHERE [LocalExecutionId] = @ExecutionId AND [StageId] > @StageId UPDATE [procfwk].[BatchExecution] SET [BatchStatus] = 'Stopping' WHERE [ExecutionId] = @ExecutionId AND [BatchStatus] = 'Running'; SET @ErrorDetail = 'Pipeline execution has an unknown status. Blocking downstream stages as a precaution.' RAISERROR(@ErrorDetail,16,1); RETURN 0; END; ELSE IF ([procfwk].[GetPropertyValueInternal]('FailureHandling')) = 'DependencyChain' BEGIN EXEC [procfwk].[SetExecutionBlockDependants] @ExecutionId = @ExecutionId, @PipelineId = @PipelineId END; ELSE BEGIN RAISERROR('Unknown failure handling state.',16,1); RETURN 0; END; END; END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/SetLogPipelineValidating.sql ================================================ CREATE PROCEDURE [procfwk].[SetLogPipelineValidating] ( @ExecutionId UNIQUEIDENTIFIER, @StageId INT, @PipelineId INT ) AS BEGIN SET NOCOUNT ON; UPDATE [procfwk].[CurrentExecution] SET [PipelineStatus] = 'Validating' WHERE [LocalExecutionId] = @ExecutionId AND [StageId] = @StageId AND [PipelineId] = @PipelineId END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/SetLogStagePreparing.sql ================================================ CREATE PROCEDURE [procfwk].[SetLogStagePreparing] ( @ExecutionId UNIQUEIDENTIFIER, @StageId INT ) AS BEGIN SET NOCOUNT ON; UPDATE [procfwk].[CurrentExecution] SET [PipelineStatus] = 'Preparing' WHERE [LocalExecutionId] = @ExecutionId AND [StageId] = @StageId AND [StartDateTime] IS NULL AND [IsBlocked] <> 1; END; ================================================ FILE: MetadataDB/procfwk/Stored Procedures/UpdateExecutionLog.sql ================================================ CREATE PROCEDURE [procfwk].[UpdateExecutionLog] ( @PerformErrorCheck BIT = 1, @ExecutionId UNIQUEIDENTIFIER = NULL ) AS BEGIN SET NOCOUNT ON; DECLARE @AllCount INT DECLARE @SuccessCount INT IF([procfwk].[GetPropertyValueInternal]('UseExecutionBatches')) = '0' BEGIN IF @PerformErrorCheck = 1 BEGIN --Check current execution SELECT @AllCount = COUNT(0) FROM [procfwk].[CurrentExecution] SELECT @SuccessCount = COUNT(0) FROM [procfwk].[CurrentExecution] WHERE [PipelineStatus] = 'Success' IF @AllCount <> @SuccessCount BEGIN RAISERROR('Framework execution complete but not all Worker pipelines succeeded. See the [procfwk].[CurrentExecution] table for details',16,1); RETURN 0; END; END; --Do this if no error raised and when called by the execution wrapper (OverideRestart = 1). INSERT INTO [procfwk].[ExecutionLog] ( [LocalExecutionId], [StageId], [PipelineId], [CallingOrchestratorName], [ResourceGroupName], [OrchestratorType], [OrchestratorName], [PipelineName], [StartDateTime], [PipelineStatus], [EndDateTime], [PipelineRunId], [PipelineParamsUsed] ) SELECT [LocalExecutionId], [StageId], [PipelineId], [CallingOrchestratorName], [ResourceGroupName], [OrchestratorType], [OrchestratorName], [PipelineName], [StartDateTime], [PipelineStatus], [EndDateTime], [PipelineRunId], [PipelineParamsUsed] FROM [procfwk].[CurrentExecution]; TRUNCATE TABLE [procfwk].[CurrentExecution]; END ELSE IF ([procfwk].[GetPropertyValueInternal]('UseExecutionBatches')) = '1' BEGIN IF @PerformErrorCheck = 1 BEGIN --Check current execution SELECT @AllCount = COUNT(0) FROM [procfwk].[CurrentExecution] WHERE [LocalExecutionId] = @ExecutionId; SELECT @SuccessCount = COUNT(0) FROM [procfwk].[CurrentExecution] WHERE [LocalExecutionId] = @ExecutionId AND [PipelineStatus] = 'Success'; IF @AllCount <> @SuccessCount BEGIN UPDATE [procfwk].[BatchExecution] SET [BatchStatus] = 'Stopped', [EndDateTime] = GETUTCDATE() WHERE [ExecutionId] = @ExecutionId; RAISERROR('Framework execution complete for batch but not all Worker pipelines succeeded. See the [procfwk].[CurrentExecution] table for details',16,1); RETURN 0; END; ELSE BEGIN UPDATE [procfwk].[BatchExecution] SET [BatchStatus] = 'Success', [EndDateTime] = GETUTCDATE() WHERE [ExecutionId] = @ExecutionId; END; END; --end check --Do this if no error raised and when called by the execution wrapper (OverideRestart = 1). INSERT INTO [procfwk].[ExecutionLog] ( [LocalExecutionId], [StageId], [PipelineId], [CallingOrchestratorName], [ResourceGroupName], [OrchestratorType], [OrchestratorName], [PipelineName], [StartDateTime], [PipelineStatus], [EndDateTime], [PipelineRunId], [PipelineParamsUsed] ) SELECT [LocalExecutionId], [StageId], [PipelineId], [CallingOrchestratorName], [ResourceGroupName], [OrchestratorType], [OrchestratorName], [PipelineName], [StartDateTime], [PipelineStatus], [EndDateTime], [PipelineRunId], [PipelineParamsUsed] FROM [procfwk].[CurrentExecution] WHERE [LocalExecutionId] = @ExecutionId; DELETE FROM [procfwk].[CurrentExecution] WHERE [LocalExecutionId] = @ExecutionId; END; END; ================================================ FILE: MetadataDB/procfwk/Synonyms/AverageStageDuration.sql ================================================ CREATE SYNONYM [procfwk].[AverageStageDuration] FOR [procfwkReporting].[AverageStageDuration]; ================================================ FILE: MetadataDB/procfwk/Synonyms/CompleteExecutionErrorLog.sql ================================================ CREATE SYNONYM [procfwk].[CompleteExecutionErrorLog] FOR [procfwkReporting].[CompleteExecutionErrorLog]; ================================================ FILE: MetadataDB/procfwk/Synonyms/CompleteExecutionLog.sql ================================================ CREATE SYNONYM [procfwk].[CompleteExecutionLog] FOR [procfwkReporting].[CompleteExecutionLog]; ================================================ FILE: MetadataDB/procfwk/Synonyms/CurrentExecutionSummary.sql ================================================ CREATE SYNONYM [procfwk].[CurrentExecutionSummary] FOR [procfwkReporting].[CurrentExecutionSummary]; ================================================ FILE: MetadataDB/procfwk/Synonyms/DataFactoryDetails.sql ================================================ CREATE SYNONYM [procfwk].[DataFactoryDetails] FOR [procfwk].[Orchestrators]; ================================================ FILE: MetadataDB/procfwk/Synonyms/LastExecution.sql ================================================ CREATE SYNONYM [procfwk].[LastExecution] FOR [procfwkReporting].[LastExecution]; ================================================ FILE: MetadataDB/procfwk/Synonyms/LastExecutionSummary.sql ================================================ CREATE SYNONYM [procfwk].[LastExecutionSummary] FOR [procfwkReporting].[LastExecutionSummary]; ================================================ FILE: MetadataDB/procfwk/Synonyms/PipelineDependencyChains.sql ================================================ CREATE SYNONYM [procfwk].[PipelineDependencyChains] FOR [procfwkHelpers].[PipelineDependencyChains]; ================================================ FILE: MetadataDB/procfwk/Synonyms/PipelineProcesses.sql ================================================ CREATE SYNONYM [procfwk].[PipelineProcesses] FOR [procfwk].[Pipelines]; ================================================ FILE: MetadataDB/procfwk/Synonyms/ProcessingStageDetails.sql ================================================ CREATE SYNONYM [procfwk].[ProcessingStageDetails] FOR [procfwk].[Stages]; ================================================ FILE: MetadataDB/procfwk/Synonyms/WorkerParallelismOverTime.sql ================================================ CREATE SYNONYM [procfwk].[WorkerParallelismOverTime] FOR [procfwkReporting].[WorkerParallelismOverTime]; ================================================ FILE: MetadataDB/procfwk/Tables/AlertOutcomes.sql ================================================ CREATE TABLE [procfwk].[AlertOutcomes] ( [OutcomeBitPosition] INT IDENTITY(0,1) NOT NULL, [PipelineOutcomeStatus] NVARCHAR(200) NOT NULL, [BitValue] AS (POWER((2),[OutcomeBitPosition])), CONSTRAINT [PK_AlertOutcomes] PRIMARY KEY CLUSTERED ([OutcomeBitPosition] ASC), CONSTRAINT [UK_PipelineOutcomeStatus] UNIQUE ([PipelineOutcomeStatus]) ) ================================================ FILE: MetadataDB/procfwk/Tables/BatchExecution.sql ================================================ CREATE TABLE [procfwk].[BatchExecution]( [BatchId] [UNIQUEIDENTIFIER] NOT NULL, [ExecutionId] [UNIQUEIDENTIFIER] NOT NULL, [BatchName] VARCHAR(255) NOT NULL, [BatchStatus] [NVARCHAR](200) NOT NULL, [StartDateTime] [DATETIME] NOT NULL, [EndDateTime] [DATETIME] NULL, CONSTRAINT [PK_BatchExecution] PRIMARY KEY CLUSTERED ( [BatchId] ASC, [ExecutionId] ASC ) ) ================================================ FILE: MetadataDB/procfwk/Tables/BatchStageLink.sql ================================================ CREATE TABLE [procfwk].[BatchStageLink] ( [BatchId] [UNIQUEIDENTIFIER] NOT NULL, [StageId] [INT] NOT NULL, CONSTRAINT [PK_BatchStageLink] PRIMARY KEY CLUSTERED ( [BatchId] ASC, [StageId] ASC ), CONSTRAINT [FK_BatchStageLink_Batches] FOREIGN KEY([BatchId]) REFERENCES [procfwk].[Batches] ([BatchId]), CONSTRAINT [FK_BatchStageLink_Stages] FOREIGN KEY([StageId]) REFERENCES [procfwk].[Stages] ([StageId]) ) ================================================ FILE: MetadataDB/procfwk/Tables/Batches.sql ================================================ CREATE TABLE [procfwk].[Batches]( [BatchId] UNIQUEIDENTIFIER DEFAULT(NEWID()) NOT NULL, [BatchName] [VARCHAR](255) NOT NULL, [BatchDescription] [VARCHAR](4000) NULL, [Enabled] [BIT] DEFAULT(0) NOT NULL, CONSTRAINT [PK_Batches] PRIMARY KEY CLUSTERED ( [BatchId] ASC ) ) ================================================ FILE: MetadataDB/procfwk/Tables/CurrentExecution.sql ================================================ CREATE TABLE [procfwk].[CurrentExecution] ( [LocalExecutionId] UNIQUEIDENTIFIER NOT NULL, [StageId] INT NOT NULL, [PipelineId] INT NOT NULL, [CallingOrchestratorName] NVARCHAR(200) NOT NULL, [ResourceGroupName]NVARCHAR (200) NOT NULL, [OrchestratorType] CHAR(3) NOT NULL, [OrchestratorName] NVARCHAR (200) NOT NULL, [PipelineName] NVARCHAR (200) NOT NULL, [StartDateTime] DATETIME NULL, [PipelineStatus] NVARCHAR (200) NULL, [LastStatusCheckDateTime] DATETIME NULL, [EndDateTime] DATETIME NULL, [IsBlocked] BIT NOT NULL DEFAULT 0, [PipelineRunId] UNIQUEIDENTIFIER NULL, [PipelineParamsUsed] NVARCHAR(MAX) NULL, CONSTRAINT [PK_CurrentExecution] PRIMARY KEY CLUSTERED ([LocalExecutionId] ASC, [StageId] ASC, [PipelineId] ASC) ); GO CREATE NONCLUSTERED INDEX [IDX_GetPipelinesInStage] ON [procfwk].[CurrentExecution] ( [LocalExecutionId], [StageId], [PipelineStatus] ) INCLUDE ( [PipelineId], [PipelineName], [OrchestratorType], [OrchestratorName], [ResourceGroupName] ) GO ================================================ FILE: MetadataDB/procfwk/Tables/ErrorLog.sql ================================================ CREATE TABLE [procfwk].[ErrorLog] ( [LogId] [int] IDENTITY(1,1) NOT NULL, [LocalExecutionId] [uniqueidentifier] NOT NULL, [PipelineRunId] [uniqueidentifier] NOT NULL, [ActivityRunId] [uniqueidentifier] NOT NULL, [ActivityName] [varchar](100) NOT NULL, [ActivityType] [varchar](100) NOT NULL, [ErrorCode] VARCHAR(100) NOT NULL, [ErrorType] [varchar](100) NOT NULL, [ErrorMessage] [nvarchar](MAX) NULL, CONSTRAINT [PK_ErrorLog] PRIMARY KEY CLUSTERED ( [LogId] ASC ) ) ================================================ FILE: MetadataDB/procfwk/Tables/ExecutionLog.sql ================================================ CREATE TABLE [procfwk].[ExecutionLog] ( [LogId] INT IDENTITY (1, 1) NOT NULL, [LocalExecutionId] UNIQUEIDENTIFIER NOT NULL, [StageId] INT NOT NULL, [PipelineId] INT NOT NULL, [CallingOrchestratorName] NVARCHAR(200) NOT NULL DEFAULT ('Unknown'), [ResourceGroupName] NVARCHAR(200) NOT NULL DEFAULT ('Unknown'), [OrchestratorType] CHAR(3) NOT NULL DEFAULT('N/A'), [OrchestratorName] NVARCHAR(200) NOT NULL DEFAULT ('Unknown'), [PipelineName] NVARCHAR (200) NOT NULL, [StartDateTime] DATETIME NULL, [PipelineStatus] NVARCHAR (200) NULL, [EndDateTime] DATETIME NULL, [PipelineRunId] UNIQUEIDENTIFIER NULL, [PipelineParamsUsed] NVARCHAR(MAX) NULL DEFAULT ('None'), CONSTRAINT [PK_ExecutionLog] PRIMARY KEY CLUSTERED ([LogId] ASC) ); ================================================ FILE: MetadataDB/procfwk/Tables/Orchestrators.sql ================================================ CREATE TABLE [procfwk].[Orchestrators] ( [OrchestratorId] [int] IDENTITY(1,1) NOT NULL, [OrchestratorName] NVARCHAR(200) NOT NULL, [OrchestratorType] CHAR(3) NOT NULL, [IsFrameworkOrchestrator] BIT NOT NULL DEFAULT(0), [ResourceGroupName] NVARCHAR(200) NOT NULL, [SubscriptionId] UNIQUEIDENTIFIER NOT NULL, [Description] NVARCHAR(MAX) NULL, CONSTRAINT [OrchestratorType] CHECK ([OrchestratorType] IN ('ADF','SYN')), CONSTRAINT [FK_Orchestrators_Subscriptions] FOREIGN KEY([SubscriptionId]) REFERENCES [procfwk].[Subscriptions] ([SubscriptionId]), CONSTRAINT [PK_Orchestrators] PRIMARY KEY CLUSTERED ([OrchestratorId] ASC) ) ================================================ FILE: MetadataDB/procfwk/Tables/PipelineAlertLink.sql ================================================ CREATE TABLE [procfwk].[PipelineAlertLink] ( [AlertId] INT IDENTITY(1,1) NOT NULL, [PipelineId] INT NOT NULL, [RecipientId] INT NOT NULL, [OutcomesBitValue] INT NOT NULL, [Enabled] BIT NOT NULL DEFAULT 1, CONSTRAINT [PK_PipelineAlertLink] PRIMARY KEY CLUSTERED ([AlertId] ASC), CONSTRAINT [FK_PipelineAlertLink_Pipelines] FOREIGN KEY([PipelineId]) REFERENCES [procfwk].[Pipelines] ([PipelineId]), CONSTRAINT [FK_PipelineAlertLink_Recipients] FOREIGN KEY([RecipientId]) REFERENCES [procfwk].[Recipients] ([RecipientId]) ); ================================================ FILE: MetadataDB/procfwk/Tables/PipelineAuthLink.sql ================================================ CREATE TABLE [procfwk].[PipelineAuthLink] ( [AuthId] [int] IDENTITY(1,1) NOT NULL, [PipelineId] [int] NOT NULL, [OrchestratorId] [int] NOT NULL, [CredentialId] [int] NOT NULL, CONSTRAINT [PK_PipelineAuthLink] PRIMARY KEY CLUSTERED ([AuthId] ASC), CONSTRAINT [FK_PipelineAuthLink_Orchestrators] FOREIGN KEY([OrchestratorId]) REFERENCES [procfwk].[Orchestrators] ([OrchestratorId]), CONSTRAINT [FK_PipelineAuthLink_Pipelines] FOREIGN KEY([PipelineId]) REFERENCES [procfwk].[Pipelines] ([PipelineId]), CONSTRAINT [FK_PipelineAuthLink_ServicePrincipals] FOREIGN KEY([CredentialId]) REFERENCES [dbo].[ServicePrincipals] ([CredentialId]) ); ================================================ FILE: MetadataDB/procfwk/Tables/PipelineDependencies.sql ================================================ CREATE TABLE [procfwk].[PipelineDependencies] ( [DependencyId] [INT] IDENTITY(1,1) NOT NULL, [PipelineId] [INT] NOT NULL, [DependantPipelineId] [INT] NOT NULL, CONSTRAINT [PK_PipelineDependencies] PRIMARY KEY CLUSTERED ([DependencyId] ASC), CONSTRAINT [FK_PipelineDependencies_Pipelines] FOREIGN KEY([PipelineId]) REFERENCES [procfwk].[Pipelines] ([PipelineId]), CONSTRAINT [FK_PipelineDependencies_Pipelines1] FOREIGN KEY([DependantPipelineId]) REFERENCES [procfwk].[Pipelines] ([PipelineId]), CONSTRAINT [UK_PipelinesToDependantPipelines] UNIQUE ([PipelineId],[DependantPipelineId]), CONSTRAINT [EQ_PipelineIdDependantPipelineId] CHECK ([PipelineId] <> [DependantPipelineId]) ) ================================================ FILE: MetadataDB/procfwk/Tables/PipelineParameters.sql ================================================ CREATE TABLE [procfwk].[PipelineParameters] ( [ParameterId] INT IDENTITY (1, 1) NOT NULL, [PipelineId] INT NOT NULL, [ParameterName] VARCHAR (128) NOT NULL, [ParameterValue] NVARCHAR(MAX) NULL, [ParameterValueLastUsed] NVARCHAR(MAX) NULL, CONSTRAINT [PK_PipelineParameters] PRIMARY KEY CLUSTERED ([ParameterId] ASC), CONSTRAINT [FK_PipelineParameters_Pipelines] FOREIGN KEY ([PipelineId]) REFERENCES [procfwk].[Pipelines] ([PipelineId]) ); ================================================ FILE: MetadataDB/procfwk/Tables/Pipelines.sql ================================================ CREATE TABLE [procfwk].[Pipelines] ( [PipelineId] INT IDENTITY (1, 1) NOT NULL, [OrchestratorId] INT NOT NULL, [StageId] INT NOT NULL, [PipelineName] NVARCHAR (200) NOT NULL, [LogicalPredecessorId] INT NULL, [Enabled] BIT CONSTRAINT [DF_Pipelines_Enabled] DEFAULT ((1)) NOT NULL, CONSTRAINT [PK_Pipelines] PRIMARY KEY CLUSTERED ([PipelineId] ASC), CONSTRAINT [FK_Pipelines_Stages] FOREIGN KEY ([StageId]) REFERENCES [procfwk].[Stages] ([StageId]), CONSTRAINT [FK_Pipelines_Orchestrators] FOREIGN KEY([OrchestratorId]) REFERENCES [procfwk].[Orchestrators] ([OrchestratorId]), CONSTRAINT [FK_Pipelines_Pipelines] FOREIGN KEY([LogicalPredecessorId]) REFERENCES [procfwk].[Pipelines] ([PipelineId]) ); ================================================ FILE: MetadataDB/procfwk/Tables/Properties.sql ================================================ CREATE TABLE [procfwk].[Properties] ( [PropertyId] [int] IDENTITY (1, 1) NOT NULL, [PropertyName] [varchar](128) NOT NULL, [PropertyValue] [nvarchar](MAX) NOT NULL, [Description] [nvarchar](MAX) NULL, [ValidFrom] [datetime] CONSTRAINT [DF_Properties_ValidFrom] DEFAULT (GETDATE()) NOT NULL, [ValidTo] [datetime] NULL, CONSTRAINT [PK_Properties] PRIMARY KEY CLUSTERED ([PropertyId] ASC, [PropertyName] ASC) ) GO ================================================ FILE: MetadataDB/procfwk/Tables/Recipients.sql ================================================ CREATE TABLE [procfwk].[Recipients] ( [RecipientId] INT IDENTITY(1,1) NOT NULL, [Name] VARCHAR(255) NULL, [EmailAddress] NVARCHAR(500) NOT NULL, [MessagePreference] CHAR(3) NOT NULL DEFAULT ('TO'), CONSTRAINT [MessagePreferenceValue] CHECK ([MessagePreference] IN ('TO','CC','BCC')), [Enabled] BIT NOT NULL DEFAULT 1, CONSTRAINT [PK_Recipients] PRIMARY KEY CLUSTERED ([RecipientId] ASC), CONSTRAINT [UK_EmailAddressMessagePreference] UNIQUE ([EmailAddress],[MessagePreference]) ); ================================================ FILE: MetadataDB/procfwk/Tables/Stages.sql ================================================ CREATE TABLE [procfwk].[Stages] ( [StageId] INT IDENTITY (1, 1) NOT NULL, [StageName] VARCHAR (225) NOT NULL, [StageDescription] VARCHAR (4000) NULL, [Enabled] BIT CONSTRAINT [DF_Stages_Enabled] DEFAULT ((1)) NOT NULL, CONSTRAINT [PK_Stages] PRIMARY KEY CLUSTERED ([StageId] ASC) ); ================================================ FILE: MetadataDB/procfwk/Tables/Subscriptions.sql ================================================ CREATE TABLE [procfwk].[Subscriptions] ( [SubscriptionId] UNIQUEIDENTIFIER NOT NULL, [Name] NVARCHAR(200) NOT NULL, [Description] NVARCHAR(MAX) NULL, [TenantId] UNIQUEIDENTIFIER NOT NULL, CONSTRAINT [PK_Subscriptions] PRIMARY KEY CLUSTERED ([SubscriptionId] ASC), CONSTRAINT [FK_Subscriptions_Tenants] FOREIGN KEY([TenantId]) REFERENCES [procfwk].[Tenants] ([TenantId]) ) ================================================ FILE: MetadataDB/procfwk/Tables/Tenants.sql ================================================ CREATE TABLE [procfwk].[Tenants] ( [TenantId] [UNIQUEIDENTIFIER] NOT NULL, [Name] [NVARCHAR](200) NOT NULL, [Description] [NVARCHAR](MAX) NULL, CONSTRAINT [PK_Tenants] PRIMARY KEY CLUSTERED ([TenantId] ASC) ) ================================================ FILE: MetadataDB/procfwk/Views/CurrentProperties.sql ================================================ CREATE VIEW [procfwk].[CurrentProperties] AS SELECT [PropertyName], [PropertyValue] FROM [procfwk].[Properties] WHERE [ValidTo] IS NULL; ================================================ FILE: MetadataDB/procfwk/Views/DataFactorys.sql ================================================ CREATE VIEW [procfwk].[DataFactorys] AS SELECT [OrchestratorId] AS DataFactoryId, [OrchestratorName] AS DataFactoryName, [ResourceGroupName], [SubscriptionId], [Description] FROM [procfwk].[Orchestrators] WHERE [OrchestratorType] = 'ADF'; ================================================ FILE: MetadataDB/procfwk/Views/PipelineParameterDataSizes.sql ================================================ CREATE VIEW [procfwk].[PipelineParameterDataSizes] AS SELECT [PipelineId], SUM( (CAST( DATALENGTH( STRING_ESCAPE([ParameterName] + [ParameterValue],'json')) AS DECIMAL) /1024) --KB /1024 --MB ) AS Size FROM [procfwk].[PipelineParameters] GROUP BY [PipelineId]; ================================================ FILE: MetadataDB/procfwkHelpers/Functions/CheckForValidURL.sql ================================================ CREATE FUNCTION [procfwkHelpers].[CheckForValidURL] (@Url NVARCHAR(MAX)) RETURNS INT AS BEGIN DECLARE @Ending VARCHAR(50) DECLARE @TempString VARCHAR(50) --check URL start IF CHARINDEX('https://', @Url) <> 1 BEGIN RETURN 0; END --check for expected sub domains IF CHARINDEX('vault.azure.net', @Url) = 0 BEGIN RETURN 0; END --check for expected value type IF CHARINDEX('secrets', @Url) = 0 BEGIN RETURN 0; END --attempt to check for secret version SELECT @Ending = CASE WHEN RIGHT(@Url,1) = '/' THEN REVERSE(LEFT(@Url,LEN(@Url)-1)) ELSE REVERSE(@Url) END, @Ending = REVERSE(LEFT(@Ending,CHARINDEX('/',@Ending)-1)) IF LEN(@Ending) = 32 BEGIN SET @TempString = SUBSTRING(@Ending, 1, 8) + '-' + SUBSTRING(@Ending, 9, 4) + '-' + SUBSTRING(@Ending, 13, 4) + '-' + SUBSTRING(@Ending, 13, 4) + '-' + SUBSTRING(@Ending, 20, 12) END IF TRY_CAST(@TempString AS UNIQUEIDENTIFIER) IS NOT NULL BEGIN RETURN 0; END; -- It is a valid URL RETURN 1; END; ================================================ FILE: MetadataDB/procfwkHelpers/Stored Procedures/AddPipelineDependant.sql ================================================ CREATE PROCEDURE [procfwkHelpers].[AddPipelineDependant] ( @PipelineName NVARCHAR(200), @DependantPipelineName NVARCHAR(200) ) AS BEGIN SET NOCOUNT ON; DECLARE @PipelineId INT; DECLARE @DependantPipelineId INT; --get pipeline ids SELECT @PipelineId = [PipelineId] FROM [procfwk].[Pipelines] WHERE [PipelineName] = @PipelineName; SELECT @DependantPipelineId = [PipelineId] FROM [procfwk].[Pipelines] WHERE [PipelineName] = @DependantPipelineName; --defensive checks IF @PipelineId IS NULL BEGIN RAISERROR('Pipeline not found in pipelines table.', 16,1); RETURN 0; END; IF @DependantPipelineId IS NULL BEGIN RAISERROR('Dependant pipeline not found in pipelines table.', 16,1); RETURN 0; END; IF @PipelineId = @DependantPipelineId BEGIN RAISERROR('Pipeline cannot be dependant on itself.', 16,1); RETURN 0; END; IF EXISTS ( SELECT * FROM [procfwk].[Pipelines] pp INNER JOIN [procfwk].[Pipelines] dp ON dp.[PipelineId] = @DependantPipelineId WHERE pp.[PipelineId] = @PipelineId AND pp.[StageId] = dp.[StageId] ) BEGIN RAISERROR('Pipeline and dependent pipeline cannot be in the same execution stage.', 16,1); RETURN 0; END; --final soft check and insert IF EXISTS ( SELECT * FROM [procfwk].[PipelineDependencies] WHERE [PipelineId] = @PipelineId AND [DependantPipelineId] = @DependantPipelineId ) BEGIN PRINT 'Dependency already exists. Nothing added.' RETURN 0; END ELSE BEGIN INSERT INTO [procfwk].[PipelineDependencies] ( [PipelineId], [DependantPipelineId] ) VALUES ( @PipelineId, @DependantPipelineId ) END; END; ================================================ FILE: MetadataDB/procfwkHelpers/Stored Procedures/AddPipelineViaPowerShell.sql ================================================ CREATE PROCEDURE [procfwkHelpers].[AddPipelineViaPowerShell] ( @ResourceGroup NVARCHAR(200), @OrchestratorName NVARCHAR(200), @OrchestratorType CHAR(3) = 'ADF', @PipelineName NVARCHAR(200) ) AS BEGIN SET NOCOUNT ON; DECLARE @OrchestratorId INT DECLARE @StageId INT DECLARE @StageName VARCHAR(255) = 'PoShAdded' --get/set orchestrator IF EXISTS ( SELECT * FROM [procfwk].[Orchestrators] WHERE [OrchestratorName] = @OrchestratorName AND [ResourceGroupName] = @ResourceGroup AND [OrchestratorType] = @OrchestratorType ) BEGIN SELECT @OrchestratorId = [OrchestratorId] FROM [procfwk].[Orchestrators] WHERE [OrchestratorName] = @OrchestratorName AND [ResourceGroupName] = @ResourceGroup AND [OrchestratorType] = @OrchestratorType; END ELSE BEGIN INSERT INTO [procfwk].[Orchestrators] ( [OrchestratorName], [OrchestratorType], [ResourceGroupName], [Description], [SubscriptionId] ) VALUES ( @OrchestratorName, @OrchestratorType, @ResourceGroup, 'Added via PowerShell.', '12345678-1234-1234-1234-012345678910' ) SELECT @OrchestratorId = SCOPE_IDENTITY(); END --get/set stage IF EXISTS ( SELECT * FROM [procfwk].[Stages] WHERE [StageName] = @StageName ) BEGIN SELECT @StageId = [StageId] FROM [procfwk].[Stages] WHERE [StageName] = @StageName; END; ELSE BEGIN INSERT INTO [procfwk].[Stages] ( [StageName], [StageDescription], [Enabled] ) VALUES ( @StageName, 'Added via PowerShell.', 1 ); SELECT @StageId = SCOPE_IDENTITY(); END; --upsert pipeline ;WITH sourceData AS ( SELECT @OrchestratorId AS OrchestratorId, @PipelineName AS PipelineName, @StageId AS StageId, NULL AS LogicalPredecessorId, 1 AS [Enabled] ) MERGE INTO [procfwk].[Pipelines] AS tgt USING sourceData AS src ON tgt.[OrchestratorId] = src.[OrchestratorId] AND tgt.[PipelineName] = src.[PipelineName] WHEN MATCHED THEN UPDATE SET tgt.[StageId] = src.[StageId], tgt.[LogicalPredecessorId] = src.[LogicalPredecessorId], tgt.[Enabled] = src.[Enabled] WHEN NOT MATCHED BY TARGET THEN INSERT ( [OrchestratorId], [StageId], [PipelineName], [LogicalPredecessorId], [Enabled] ) VALUES ( src.[OrchestratorId], src.[StageId], src.[PipelineName], src.[LogicalPredecessorId], src.[Enabled] ); END; ================================================ FILE: MetadataDB/procfwkHelpers/Stored Procedures/AddProperty.sql ================================================ CREATE PROCEDURE [procfwkHelpers].[AddProperty] ( @PropertyName VARCHAR(128), @PropertyValue NVARCHAR(MAX), @Description NVARCHAR(MAX) = NULL ) AS BEGIN SET NOCOUNT ON; --defensive check IF EXISTS ( SELECT * FROM [procfwk].[Properties] WHERE [PropertyName] = @PropertyName AND [ValidTo] IS NOT NULL ) AND NOT EXISTS ( SELECT * FROM [procfwk].[Properties] WHERE [PropertyName] = @PropertyName AND [ValidTo] IS NULL ) BEGIN WITH lastValue AS ( SELECT [PropertyId], ROW_NUMBER() OVER (PARTITION BY [PropertyName] ORDER BY [ValidTo] ASC) AS Rn FROM [procfwk].[Properties] WHERE [PropertyName] = @PropertyName ) --reset property if valid to date has been incorrectly set UPDATE prop SET [ValidTo] = NULL FROM [procfwk].[Properties] prop INNER JOIN lastValue ON prop.[PropertyId] = lastValue.[PropertyId] WHERE lastValue.[Rn] = 1 END --upsert property ;WITH sourceTable AS ( SELECT @PropertyName AS PropertyName, @PropertyValue AS PropertyValue, @Description AS [Description], GETUTCDATE() AS StartEndDate ) --insert new version of existing property from MERGE OUTPUT INSERT INTO [procfwk].[Properties] ( [PropertyName], [PropertyValue], [Description], [ValidFrom] ) SELECT [PropertyName], [PropertyValue], [Description], GETUTCDATE() FROM ( MERGE INTO [procfwk].[Properties] targetTable USING sourceTable ON sourceTable.[PropertyName] = targetTable.[PropertyName] --set valid to date on existing property WHEN MATCHED AND [ValidTo] IS NULL THEN UPDATE SET targetTable.[ValidTo] = sourceTable.[StartEndDate] --add new property WHEN NOT MATCHED BY TARGET THEN INSERT ( [PropertyName], [PropertyValue], [Description], [ValidFrom] ) VALUES ( sourceTable.[PropertyName], sourceTable.[PropertyValue], sourceTable.[Description], sourceTable.[StartEndDate] ) --for new entry of existing record OUTPUT $action AS [Action], sourceTable.* ) AS MergeOutput WHERE MergeOutput.[Action] = 'UPDATE'; END; ================================================ FILE: MetadataDB/procfwkHelpers/Stored Procedures/AddRecipientPipelineAlerts.sql ================================================ CREATE PROCEDURE [procfwkHelpers].[AddRecipientPipelineAlerts] ( @RecipientName VARCHAR(255), @PipelineName NVARCHAR(200) = NULL, @AlertForStatus NVARCHAR(500) = 'All' ) AS BEGIN SET NOCOUNT ON; DECLARE @ActualBitValue INT DECLARE @SQL NVARCHAR(MAX) = '' DECLARE @BitValue TABLE ([TotalBitValue] INT NOT NULL); --get alert status bit value SET @AlertForStatus = LTRIM(RTRIM(@AlertForStatus)) SET @AlertForStatus = REPLACE(@AlertForStatus,' ','') SET @AlertForStatus = '''' + REPLACE(@AlertForStatus,',',''',''') + '''' SET @SQL = ' SELECT SUM([BitValue]) AS ''TotalBitValue'' FROM [procfwk].[AlertOutcomes] WHERE [PipelineOutcomeStatus] IN (' + @AlertForStatus + ') ' INSERT INTO @BitValue ([TotalBitValue]) EXECUTE(@SQL) SELECT @ActualBitValue = [TotalBitValue] FROM @BitValue --set link table IF @PipelineName IS NOT NULL BEGIN --add alert for specific pipeline if doesn't exist INSERT INTO [procfwk].[PipelineAlertLink] ( [PipelineId], [RecipientId], [OutcomesBitValue] ) SELECT p.[PipelineId], r.[RecipientId], @ActualBitValue FROM [procfwk].[Pipelines] p INNER JOIN [procfwk].[Recipients] r ON r.[Name] = @RecipientName LEFT OUTER JOIN [procfwk].[PipelineAlertLink] al ON p.[PipelineId] = al.[PipelineId] AND r.[RecipientId] = al.[RecipientId] WHERE p.[PipelineName] = @PipelineName AND al.[PipelineId] IS NULL AND al.[RecipientId] IS NULL; END ELSE IF @PipelineName IS NULL BEGIN --remove and re-add alerts for all pipelines DELETE al FROM [procfwk].[PipelineAlertLink] al INNER JOIN [procfwk].[Recipients] r ON al.[RecipientId] = r.[RecipientId] WHERE r.[Name] = @RecipientName; INSERT INTO [procfwk].[PipelineAlertLink] ( [PipelineId], [RecipientId], [OutcomesBitValue] ) SELECT p.[PipelineId], r.[RecipientId], @ActualBitValue FROM [procfwk].[Recipients] r CROSS JOIN [procfwk].[Pipelines] p WHERE r.[Name] = @RecipientName; END; END ================================================ FILE: MetadataDB/procfwkHelpers/Stored Procedures/AddServicePrincipal.sql ================================================ CREATE PROCEDURE [procfwkHelpers].[AddServicePrincipal] ( @OrchestratorName NVARCHAR(200), @OrchestratorType CHAR(3), @PrincipalId NVARCHAR(256), @PrincipalSecret NVARCHAR(MAX), @SpecificPipelineName NVARCHAR(200) = NULL, @PrincipalName NVARCHAR(256) = NULL ) AS BEGIN SET NOCOUNT ON; DECLARE @ErrorDetails NVARCHAR(500) = '' DECLARE @CredentialId INT DECLARE @LocalPrincipalId UNIQUEIDENTIFIER --defensive checks BEGIN TRY SELECT --assigned to variable just to supress output of SELECT @LocalPrincipalId = CAST(@PrincipalId AS UNIQUEIDENTIFIER) END TRY BEGIN CATCH SET @ErrorDetails = 'Invalid @PrincipalId provided. The format must be a UNIQUEIDENTIFIER.' RAISERROR(@ErrorDetails, 16, 1); RETURN 0; END CATCH IF NOT EXISTS ( SELECT [OrchestratorName] FROM [procfwk].[Orchestrators] WHERE [OrchestratorName] = @OrchestratorName AND [OrchestratorType] = @OrchestratorType ) BEGIN SET @ErrorDetails = 'Invalid Orchestrator name. Please ensure the Orchestrator metadata exists before trying to add authentication for it.' RAISERROR(@ErrorDetails, 16, 1); RETURN 0; END IF EXISTS ( SELECT * FROM [procfwk].[PipelineAuthLink] AL INNER JOIN [procfwk].[Orchestrators] DF ON AL.[OrchestratorId] = DF.[OrchestratorId] INNER JOIN [procfwk].[Pipelines] PP ON AL.[PipelineId] = PP.[PipelineId] WHERE DF.[OrchestratorName] = @OrchestratorName AND DF.[OrchestratorType] = @OrchestratorType AND PP.[PipelineName] = @SpecificPipelineName ) BEGIN SET @ErrorDetails = 'The provided Pipeline or Orchestrator combination already have a Service Principal. Delete the existing record using the procedure [procfwk].[DeleteServicePrincipal].' RAISERROR(@ErrorDetails, 16, 1); RETURN 0; END --add SPN for specific pipeline IF @SpecificPipelineName IS NOT NULL BEGIN --secondary defensive check for pipeline optional param IF NOT EXISTS ( SELECT [PipelineName] FROM [procfwk].[Pipelines] WHERE [PipelineName] = @SpecificPipelineName ) BEGIN SET @ErrorDetails = 'Invalid Pipeline name. Please ensure the Pipeline metadata exists before trying to add authentication for it.' RAISERROR(@ErrorDetails, 16, 1); RETURN 0; END --spn may already exist for other pipelines IF NOT EXISTS ( SELECT [PrincipalId] FROM [dbo].[ServicePrincipals] WHERE [PrincipalId] = @PrincipalId ) BEGIN --add service principal INSERT INTO [dbo].[ServicePrincipals] ( [PrincipalName], [PrincipalId], [PrincipalSecret] ) SELECT ISNULL(@PrincipalName, 'Unknown'), @PrincipalId, ENCRYPTBYPASSPHRASE(CONCAT(@OrchestratorName, @OrchestratorType, @SpecificPipelineName), @PrincipalSecret) SET @CredentialId = SCOPE_IDENTITY() END ELSE BEGIN SELECT @CredentialId = [CredentialId] FROM [dbo].[ServicePrincipals] WHERE [PrincipalId] = @PrincipalId END --add single pipeline to SPN link INSERT INTO [procfwk].[PipelineAuthLink] ( [PipelineId], [OrchestratorId], [CredentialId] ) SELECT P.[PipelineId], D.[OrchestratorId], @CredentialId FROM [procfwk].[Pipelines] P INNER JOIN [procfwk].[Orchestrators] D ON P.[OrchestratorId] = D.[OrchestratorId] WHERE P.[PipelineName] = @SpecificPipelineName AND D.[OrchestratorType] = @OrchestratorType AND D.[OrchestratorName] = @OrchestratorName; END ELSE --add SPN for all pipelines in Orchestrator BEGIN --add service principal INSERT INTO [dbo].[ServicePrincipals] ( [PrincipalName], [PrincipalId], [PrincipalSecret] ) SELECT ISNULL(@PrincipalName, 'Unknown'), @PrincipalId, ENCRYPTBYPASSPHRASE(CONCAT(@OrchestratorName, @OrchestratorType), @PrincipalSecret) SET @CredentialId = SCOPE_IDENTITY() --add link INSERT INTO [procfwk].[PipelineAuthLink] ( [PipelineId], [OrchestratorId], [CredentialId] ) SELECT P.[PipelineId], D.[OrchestratorId], @CredentialId FROM [procfwk].[Pipelines] P INNER JOIN [procfwk].[Orchestrators] D ON P.[OrchestratorId] = D.[OrchestratorId] LEFT OUTER JOIN [procfwk].[PipelineAuthLink] L ON P.[PipelineId] = L.[PipelineId] WHERE D.[OrchestratorName] = @OrchestratorName AND D.[OrchestratorType] = @OrchestratorType AND L.[PipelineId] IS NULL; END END; ================================================ FILE: MetadataDB/procfwkHelpers/Stored Procedures/AddServicePrincipalUrls.sql ================================================ CREATE PROCEDURE [procfwkHelpers].[AddServicePrincipalUrls] ( @OrchestratorName NVARCHAR(200), @OrchestratorType CHAR(3), @PrincipalIdUrl NVARCHAR(MAX), @PrincipalSecretUrl NVARCHAR(MAX), @SpecificPipelineName NVARCHAR(200) = NULL, @PrincipalName NVARCHAR(256) = NULL ) AS BEGIN SET NOCOUNT ON; DECLARE @ErrorDetails NVARCHAR(500) = '' DECLARE @CredentialId INT --defensive checks IF NOT EXISTS ( SELECT [OrchestratorName] FROM [procfwk].[Orchestrators] WHERE [OrchestratorName] = @OrchestratorName AND [OrchestratorType] = @OrchestratorType ) BEGIN SET @ErrorDetails = 'Invalid Orchestrator name. Please ensure the Orchestrator metadata exists before trying to add authentication for it.' RAISERROR(@ErrorDetails, 16, 1); RETURN 0; END IF EXISTS ( SELECT * FROM [procfwk].[PipelineAuthLink] AL INNER JOIN [procfwk].[Orchestrators] DF ON AL.[OrchestratorId] = DF.[OrchestratorId] INNER JOIN [procfwk].[Pipelines] PP ON AL.[PipelineId] = PP.[PipelineId] WHERE DF.[OrchestratorName] = @OrchestratorName AND DF.[OrchestratorType] = @OrchestratorType AND PP.[PipelineName] = @SpecificPipelineName ) BEGIN SET @ErrorDetails = 'The provided Pipeline or Orchestrator combination already have a Service Principal. Delete the existing record using the procedure [procfwk].[DeleteServicePrincipal].' RAISERROR(@ErrorDetails, 16, 1); RETURN 0; END IF ([procfwkHelpers].[CheckForValidURL](@PrincipalIdUrl)) = 0 BEGIN SET @ErrorDetails = 'PrincipalIdUrl value is not in the expected format. . Please confirm the URL follows the structure https://{YourKeyVaultName}.vault.azure.net/secrets/{YourSecretName} and does not include the secret version guid.' PRINT @ErrorDetails; END IF ([procfwkHelpers].[CheckForValidURL](@PrincipalSecretUrl)) = 0 BEGIN SET @ErrorDetails = 'PrincipalSecretUrl value is not in the expected format. Please confirm the URL follows the structure https://{YourKeyVaultName}.vault.azure.net/secrets/{YourSecretName} and does not include the secret version guid.' PRINT @ErrorDetails; END --add SPN for specific pipeline IF @SpecificPipelineName IS NOT NULL BEGIN --secondary defensive check for pipeline optional param IF NOT EXISTS ( SELECT [PipelineName] FROM [procfwk].[Pipelines] WHERE [PipelineName] = @SpecificPipelineName ) BEGIN SET @ErrorDetails = 'Invalid Pipeline name. Please ensure the Pipeline metadata exists before trying to add authentication for it.' RAISERROR(@ErrorDetails, 16, 1); RETURN 0; END --spn may already exist for other pipelines IF NOT EXISTS ( SELECT [PrincipalIdUrl] FROM [dbo].[ServicePrincipals] WHERE [PrincipalIdUrl] = @PrincipalIdUrl ) BEGIN --add service principal INSERT INTO [dbo].[ServicePrincipals] ( [PrincipalName], [PrincipalIdUrl], [PrincipalSecretUrl] ) SELECT ISNULL(@PrincipalName, 'Unknown'), @PrincipalIdUrl, @PrincipalSecretUrl SET @CredentialId = SCOPE_IDENTITY() END ELSE BEGIN SELECT @CredentialId = [CredentialId] FROM [dbo].[ServicePrincipals] WHERE [PrincipalIdUrl] = @PrincipalIdUrl END --add single pipeline to SPN link INSERT INTO [procfwk].[PipelineAuthLink] ( [PipelineId], [OrchestratorId], [CredentialId] ) SELECT P.[PipelineId], D.[OrchestratorId], @CredentialId FROM [procfwk].[Pipelines] P INNER JOIN [procfwk].[Orchestrators] D ON P.[OrchestratorId] = D.[OrchestratorId] WHERE P.[PipelineName] = @SpecificPipelineName AND D.[OrchestratorType] = @OrchestratorType AND D.[OrchestratorName] = @OrchestratorName; END ELSE --add SPN for all pipelines in Orchestrator BEGIN --add service principal INSERT INTO [dbo].[ServicePrincipals] ( [PrincipalName], [PrincipalIdUrl], [PrincipalSecretUrl] ) SELECT ISNULL(@PrincipalName, 'Unknown'), @PrincipalIdUrl, @PrincipalSecretUrl SET @CredentialId = SCOPE_IDENTITY() --add link INSERT INTO [procfwk].[PipelineAuthLink] ( [PipelineId], [OrchestratorId], [CredentialId] ) SELECT P.[PipelineId], D.[OrchestratorId], @CredentialId FROM [procfwk].[Pipelines] P INNER JOIN [procfwk].[Orchestrators] D ON P.[OrchestratorId] = D.[OrchestratorId] LEFT OUTER JOIN [procfwk].[PipelineAuthLink] L ON P.[PipelineId] = L.[PipelineId] WHERE D.[OrchestratorName] = @OrchestratorName AND D.[OrchestratorType] = @OrchestratorType AND L.[PipelineId] IS NULL; END END; ================================================ FILE: MetadataDB/procfwkHelpers/Stored Procedures/AddServicePrincipalWrapper.sql ================================================ CREATE PROCEDURE [procfwkHelpers].[AddServicePrincipalWrapper] ( @OrchestratorName NVARCHAR(200), @OrchestratorType CHAR(3), @PrincipalIdValue NVARCHAR(MAX), @PrincipalSecretValue NVARCHAR(MAX), @SpecificPipelineName NVARCHAR(200) = NULL, @PrincipalName NVARCHAR(256) = NULL ) AS BEGIN IF ([procfwk].[GetPropertyValueInternal]('SPNHandlingMethod')) = 'StoreInDatabase' BEGIN EXEC [procfwk].[AddServicePrincipal] @OrchestratorName = @OrchestratorName, @OrchestratorType = @OrchestratorType, @PrincipalId = @PrincipalIdValue, @PrincipalSecret = @PrincipalSecretValue, @PrincipalName = @PrincipalName, @SpecificPipelineName = @SpecificPipelineName END ELSE IF ([procfwk].[GetPropertyValueInternal]('SPNHandlingMethod')) = 'StoreInKeyVault' BEGIN EXEC [procfwk].[AddServicePrincipalUrls] @OrchestratorName = @OrchestratorName, @OrchestratorType = @OrchestratorType, @PrincipalIdUrl = @PrincipalIdValue, @PrincipalSecretUrl = @PrincipalSecretValue, @PrincipalName = @PrincipalName, @SpecificPipelineName = @SpecificPipelineName END ELSE BEGIN RAISERROR('Unknown SPN insert method.',16,1); RETURN 0; END END; ================================================ FILE: MetadataDB/procfwkHelpers/Stored Procedures/CheckStageAndPiplineIntegrity.sql ================================================ CREATE PROCEDURE [procfwkHelpers].[CheckStageAndPiplineIntegrity] AS BEGIN DECLARE @TempCheckStageAndPiplineIntegrity TABLE ( [ResourceGroupName] NVARCHAR(200) NOT NULL, [OrchestratorName] NVARCHAR(200) NOT NULL, [StageId] INT NOT NULL, [StageName] VARCHAR(225) NOT NULL, [PipelineId] INT NOT NULL, [PipelineName] NVARCHAR(200) NOT NULL, [Enabled] BIT NOT NULL, [SuccessorStageId] INT NULL, [SuccessorStage] VARCHAR(225) NULL, [SuccessorId] INT NULL, [SuccessorName] NVARCHAR(200) NULL, [Information] VARCHAR(92) NULL ) --get min execution stage ;WITH firstStage AS ( SELECT MIN([StageId]) AS firstStageId FROM [procfwk].[Stages] ) --query metadata INSERT INTO @TempCheckStageAndPiplineIntegrity SELECT adf.[ResourceGroupName], adf.[OrchestratorName], base.[StageId], baseStage.[StageName], base.[PipelineId], base.[PipelineName], base.[Enabled], preds.[StageId] AS SuccessorStageId, predsStage.[StageName] AS SuccessorStage, preds.[PipelineId] AS SuccessorId, preds.[PipelineName] AS SuccessorName, CASE WHEN preds.[StageId] > base.[StageId] +1 THEN 'Successor pipeline could be moved to an earlier stage.' WHEN preds.[StageId] = base.[StageId] THEN 'Dependency issue, predeccessor pipeline is currently running in the same stage as successor.' WHEN preds.[PipelineId] IS NOT NULL AND base.[Enabled] = 0 THEN 'Disabled pipeline has downstream successors.' WHEN preds.[PipelineId] IS NOT NULL AND baseStage.[Enabled] = 0 THEN 'Disabled stage has downstream successors.' WHEN base.[LogicalPredecessorId] IS NULL AND base.[StageId] <> firstStage.[firstStageId] THEN 'Pipeline could be moved to an earlier stage.' ELSE NULL END AS Information FROM --get base pipeline details [procfwk].[Pipelines] base INNER JOIN [procfwk].[Orchestrators] adf ON base.[OrchestratorId] = adf.[OrchestratorId] INNER JOIN [procfwk].[Stages] baseStage ON base.[StageId] = baseStage.[StageId] --get successor details LEFT OUTER JOIN [procfwk].[Pipelines] preds ON base.[PipelineId] = preds.[LogicalPredecessorId] LEFT OUTER JOIN [procfwk].[Stages] predsStage ON preds.[StageId] = predsStage.[StageId] --other details for checking CROSS JOIN firstStage --provide outcome IF EXISTS ( SELECT [Information] FROM @TempCheckStageAndPiplineIntegrity ) BEGIN SELECT * FROM @TempCheckStageAndPiplineIntegrity END ELSE BEGIN PRINT 'No pipeline integrity issues to report. Nice work! :-)' END END; ================================================ FILE: MetadataDB/procfwkHelpers/Stored Procedures/DeleteMetadataWithIntegrity.sql ================================================ CREATE PROCEDURE [procfwkHelpers].[DeleteMetadataWithIntegrity] AS BEGIN /* DELETE ORDER IMPORTANT FOR REFERENTIAL INTEGRITY */ --BatchExecution IF OBJECT_ID(N'[procfwk].[BatchExecution]') IS NOT NULL BEGIN TRUNCATE TABLE [procfwk].[BatchExecution]; END; --CurrentExecution IF OBJECT_ID(N'[procfwk].[CurrentExecution]') IS NOT NULL BEGIN TRUNCATE TABLE [procfwk].[CurrentExecution]; END; --ExecutionLog IF OBJECT_ID(N'[procfwk].[ExecutionLog]') IS NOT NULL BEGIN TRUNCATE TABLE [procfwk].[ExecutionLog]; END --ErrorLog IF OBJECT_ID(N'[procfwk].[ExecutionLog]') IS NOT NULL BEGIN TRUNCATE TABLE [procfwk].[ErrorLog]; END --BatchStageLink IF OBJECT_ID(N'[procfwk].[BatchStageLink]') IS NOT NULL BEGIN DELETE FROM [procfwk].[BatchStageLink]; END; --Batches IF OBJECT_ID(N'[procfwk].[Batches]') IS NOT NULL BEGIN DELETE FROM [procfwk].[Batches]; END; --PipelineDependencies IF OBJECT_ID(N'[procfwk].[PipelineDependencies]') IS NOT NULL BEGIN DELETE FROM [procfwk].[PipelineDependencies]; DBCC CHECKIDENT ('[procfwk].[PipelineDependencies]', RESEED, 0); END; --PipelineAlertLink IF OBJECT_ID(N'[procfwk].[PipelineAlertLink]') IS NOT NULL BEGIN DELETE FROM [procfwk].[PipelineAlertLink]; DBCC CHECKIDENT ('[procfwk].[PipelineAlertLink]', RESEED, 0); END; --Recipients IF OBJECT_ID(N'[procfwk].[Recipients]') IS NOT NULL BEGIN DELETE FROM [procfwk].[Recipients]; DBCC CHECKIDENT ('[procfwk].[Recipients]', RESEED, 0); END; --AlertOutcomes IF OBJECT_ID(N'[procfwk].[AlertOutcomes]') IS NOT NULL BEGIN TRUNCATE TABLE [procfwk].[AlertOutcomes]; END; --PipelineAuthLink IF OBJECT_ID(N'[procfwk].[PipelineAuthLink]') IS NOT NULL BEGIN DELETE FROM [procfwk].[PipelineAuthLink]; DBCC CHECKIDENT ('[procfwk].[PipelineAuthLink]', RESEED, 0); END; --ServicePrincipals IF OBJECT_ID(N'[dbo].[ServicePrincipals]') IS NOT NULL BEGIN DELETE FROM [dbo].[ServicePrincipals]; DBCC CHECKIDENT ('[dbo].[ServicePrincipals]', RESEED, 0); END; --Properties IF OBJECT_ID(N'[procfwk].[Properties]') IS NOT NULL BEGIN DELETE FROM [procfwk].[Properties]; DBCC CHECKIDENT ('[procfwk].[Properties]', RESEED, 0); END; --PipelineParameters IF OBJECT_ID(N'[procfwk].[PipelineParameters]') IS NOT NULL BEGIN DELETE FROM [procfwk].[PipelineParameters]; DBCC CHECKIDENT ('[procfwk].[PipelineParameters]', RESEED, 0); END; --Pipelines IF OBJECT_ID(N'[procfwk].[Pipelines]') IS NOT NULL BEGIN DELETE FROM [procfwk].[Pipelines]; DBCC CHECKIDENT ('[procfwk].[Pipelines]', RESEED, 0); END; --Orchestrators IF OBJECT_ID(N'[procfwk].[Orchestrators]') IS NOT NULL BEGIN DELETE FROM [procfwk].[Orchestrators]; DBCC CHECKIDENT ('[procfwk].[Orchestrators]', RESEED, 0); END; --Stages IF OBJECT_ID(N'[procfwk].[Stages]') IS NOT NULL BEGIN DELETE FROM [procfwk].[Stages]; DBCC CHECKIDENT ('[procfwk].[Stages]', RESEED, 0); END; --Subscriptions IF OBJECT_ID(N'[procfwk].[Subscriptions]') IS NOT NULL BEGIN DELETE FROM [procfwk].[Subscriptions]; END; --Tenants IF OBJECT_ID(N'[procfwk].[Tenants]') IS NOT NULL BEGIN DELETE FROM [procfwk].[Tenants]; END; END; ================================================ FILE: MetadataDB/procfwkHelpers/Stored Procedures/DeleteMetadataWithoutIntegrity.sql ================================================ CREATE PROCEDURE [procfwkHelpers].[DeleteMetadataWithoutIntegrity] AS BEGIN DECLARE @SQL NVARCHAR(MAX) = '' ;WITH procfwkTables AS ( SELECT QUOTENAME(s.[name]) + '.' + QUOTENAME(o.[name]) AS FullName FROM sys.objects o INNER JOIN sys.schemas s ON o.[schema_id] = s.[schema_id] WHERE s.[name] LIKE 'procfwk%' AND o.[type] = 'U' UNION SELECT QUOTENAME(s.[name]) + '.' + QUOTENAME(o.[name]) AS FullName FROM sys.objects o INNER JOIN sys.schemas s ON o.[schema_id] = s.[schema_id] WHERE o.[name] = 'ServicePrincipals' AND o.[type] = 'U' ) SELECT --tables must exist or wouldnt appear in sys.objects query @SQL += 'DELETE FROM ' + [FullName] + ';' + CHAR(13) FROM procfwkTables EXEC(@SQL); END; ================================================ FILE: MetadataDB/procfwkHelpers/Stored Procedures/DeleteRecipientAlerts.sql ================================================ CREATE PROCEDURE [procfwkHelpers].[DeleteRecipientAlerts] ( @EmailAddress NVARCHAR(500), @SoftDeleteOnly BIT = 1 ) AS BEGIN SET NOCOUNT ON; --defensive check IF NOT EXISTS ( SELECT [RecipientId] FROM [procfwk].[Recipients] WHERE [EmailAddress] = @EmailAddress ) BEGIN RAISERROR('Recipient email address does not exists in [procfwk].[Recipients] table.',16,1); RETURN 0; END; --update/delete IF @SoftDeleteOnly = 1 BEGIN --disable links UPDATE al SET al.[Enabled] = 0 FROM [procfwk].[PipelineAlertLink] al INNER JOIN [procfwk].[Recipients] r ON al.[RecipientId] = r.[RecipientId] WHERE r.[EmailAddress] = @EmailAddress; --disable recipient(s) UPDATE [procfwk].[Recipients] SET [Enabled] = 0 WHERE [EmailAddress] = @EmailAddress; END ELSE BEGIN --delete links DELETE al FROM [procfwk].[PipelineAlertLink] al INNER JOIN [procfwk].[Recipients] r ON al.[RecipientId] = r.[RecipientId] WHERE r.[EmailAddress] = @EmailAddress; --delete recipient(s) DELETE FROM [procfwk].[Recipients] WHERE [EmailAddress] = @EmailAddress; END; END; ================================================ FILE: MetadataDB/procfwkHelpers/Stored Procedures/DeleteServicePrincipal.sql ================================================ CREATE PROCEDURE [procfwkHelpers].[DeleteServicePrincipal] ( @OrchestratorName NVARCHAR(200), @OrchestratorType CHAR(3), @PrincipalIdValue NVARCHAR(256), @SpecificPipelineName NVARCHAR(200) = NULL ) AS BEGIN SET NOCOUNT ON; DECLARE @ErrorDetails NVARCHAR(500) = '' DECLARE @CredentialId INT --resolve principal Id or Url to credential Id IF ([procfwk].[GetPropertyValueInternal]('SPNHandlingMethod')) = 'StoreInDatabase' BEGIN --defensive checks BEGIN TRY DECLARE @LocalPrincipalId UNIQUEIDENTIFIER SELECT --assigned to variable just to supress output of SELECT @LocalPrincipalId = CAST(@PrincipalIdValue AS UNIQUEIDENTIFIER) END TRY BEGIN CATCH SET @ErrorDetails = 'Invalid @PrincipalId provided. The format must be a UNIQUEIDENTIFIER.' RAISERROR(@ErrorDetails, 16, 1); RETURN 0; END CATCH --get cred id using principal id SELECT @CredentialId = [CredentialId] FROM [dbo].[ServicePrincipals] WHERE [PrincipalId] = @PrincipalIdValue END ELSE IF ([procfwk].[GetPropertyValueInternal]('SPNHandlingMethod')) = 'StoreInKeyVault' BEGIN --get cred id using principal id url SELECT @CredentialId = [CredentialId] FROM [dbo].[ServicePrincipals] WHERE [PrincipalIdUrl] = @PrincipalIdValue END; ELSE BEGIN RAISERROR('Unknown SPN deletion method.',16,1); RETURN 0; END; --secondary defensive checks IF NOT EXISTS ( SELECT [OrchestratorName] FROM [procfwk].[Orchestrators] WHERE [OrchestratorName] = @OrchestratorName AND [OrchestratorType] = @OrchestratorType ) BEGIN SET @ErrorDetails = 'Invalid Orchestrator name. Please ensure the Orchestrator metadata exists.' RAISERROR(@ErrorDetails, 16, 1); RETURN 0; END IF @CredentialId IS NULL BEGIN SET @ErrorDetails = 'Invalid Service Principal Id Value provided. Please ensure the Service Principal exists.' RAISERROR(@ErrorDetails, 16, 1); RETURN 0; END --delete SPN for specific pipeline IF @SpecificPipelineName IS NOT NULL BEGIN IF NOT EXISTS ( SELECT [PipelineName] FROM [procfwk].[Pipelines] WHERE [PipelineName] = @SpecificPipelineName ) BEGIN SET @ErrorDetails = 'Invalid Pipeline name. Please ensure the Pipeline metadata exists.' RAISERROR(@ErrorDetails, 16, 1); RETURN 0; END --delete links DELETE L FROM [procfwk].[PipelineAuthLink] L INNER JOIN [procfwk].[Pipelines] P ON L.[PipelineId] = P.[PipelineId] INNER JOIN [procfwk].[Orchestrators] D ON P.[OrchestratorId] = D.[OrchestratorId] AND L.[OrchestratorId] = D.[OrchestratorId] INNER JOIN [dbo].[ServicePrincipals] S ON L.[CredentialId] = S.[CredentialId] WHERE P.[PipelineName] = @SpecificPipelineName AND D.[OrchestratorName] = @OrchestratorName AND D.[OrchestratorType] = @OrchestratorType AND S.[CredentialId] = @CredentialId; END ELSE BEGIN --delete links DELETE L FROM [procfwk].[PipelineAuthLink] L INNER JOIN [procfwk].[Orchestrators] D ON L.[OrchestratorId] = D.[OrchestratorId] INNER JOIN [dbo].[ServicePrincipals] S ON L.[CredentialId] = S.[CredentialId] WHERE D.[OrchestratorName] = @OrchestratorName AND D.[OrchestratorType] = @OrchestratorType AND S.[CredentialId] = @CredentialId; END --finall, delete principal only if not still used by other pipelines DELETE SP FROM [dbo].[ServicePrincipals] SP LEFT OUTER JOIN [procfwk].[PipelineAuthLink] AL ON SP.[CredentialId] = AL.[CredentialId] WHERE SP.[CredentialId] = @CredentialId AND AL.[CredentialId] IS NULL; END; ================================================ FILE: MetadataDB/procfwkHelpers/Stored Procedures/GetExecutionDetails.sql ================================================ CREATE PROCEDURE [procfwkHelpers].[GetExecutionDetails] ( @LocalExecutionId UNIQUEIDENTIFIER = NULL ) AS BEGIN --Get last execution ID IF @LocalExecutionId IS NULL BEGIN WITH maxLog AS ( SELECT MAX([LogId]) AS MaxLogId FROM [procfwk].[ExecutionLog] ) SELECT @LocalExecutionId = el1.[LocalExecutionId] FROM [procfwk].[ExecutionLog] el1 INNER JOIN maxLog ON maxLog.[MaxLogId] = el1.[LogId]; END; --Execution Summary SELECT CAST(el2.[StageId] AS VARCHAR(5)) + ' - ' + stgs.[StageName] AS Stage, COUNT(0) AS RecordCount, DATEDIFF(MINUTE, MIN(el2.[StartDateTime]), MAX(el2.[EndDateTime])) DurationMinutes FROM [procfwk].[ExecutionLog] el2 INNER JOIN [procfwk].[Stages] stgs ON el2.[StageId] = stgs.[StageId] WHERE el2.[LocalExecutionId] = @LocalExecutionId GROUP BY CAST(el2.[StageId] AS VARCHAR(5)) + ' - ' + stgs.[StageName] ORDER BY CAST(el2.[StageId] AS VARCHAR(5)) + ' - ' + stgs.[StageName]; --Full execution details SELECT el3.[LogId], el3.[LocalExecutionId], el3.[OrchestratorType], el3.[OrchestratorName], el3.[StageId], stgs.[StageName], el3.[PipelineId], el3.[PipelineName], el3.[StartDateTime], el3.[EndDateTime], ISNULL(DATEDIFF(MINUTE, el3.[StartDateTime], el3.[EndDateTime]),0) AS DurationMinutes, el3.[PipelineStatus], el3.[PipelineRunId], el3.[PipelineParamsUsed], errLog.[ActivityRunId], errLog.[ActivityName], errLog.[ActivityType], errLog.[ErrorCode], errLog.[ErrorType], errLog.[ErrorMessage] FROM [procfwk].[ExecutionLog] el3 LEFT OUTER JOIN [procfwk].[ErrorLog] errLog ON el3.[LocalExecutionId] = errLog.[LocalExecutionId] AND el3.[PipelineRunId] = errLog.[PipelineRunId] INNER JOIN [procfwk].[Stages] stgs ON el3.[StageId] = stgs.[StageId] WHERE el3.[LocalExecutionId] = @LocalExecutionId ORDER BY el3.[PipelineId], el3.[StartDateTime]; END; ================================================ FILE: MetadataDB/procfwkHelpers/Stored Procedures/GetServicePrincipal.sql ================================================ CREATE PROCEDURE [procfwkHelpers].[GetServicePrincipal] ( @OrchestratorName NVARCHAR(200), @OrchestratorType CHAR(3), @PipelineName NVARCHAR(200) = NULL ) AS BEGIN SET NOCOUNT ON; DECLARE @Id NVARCHAR(MAX) DECLARE @Secret NVARCHAR(MAX) IF ([procfwk].[GetPropertyValueInternal]('SPNHandlingMethod')) = 'StoreInDatabase' BEGIN --get auth details regardless of being pipeline specific and regardless of a pipeline param being passed ;WITH cte AS ( SELECT DISTINCT S.[PrincipalId] AS Id, CAST(DECRYPTBYPASSPHRASE(CONCAT(@OrchestratorName, @PipelineName), S.[PrincipalSecret]) AS NVARCHAR(MAX)) AS [Secret] FROM [dbo].[ServicePrincipals] S INNER JOIN [procfwk].[PipelineAuthLink] L ON S.[CredentialId] = L.[CredentialId] INNER JOIN [procfwk].[Pipelines] P ON L.[PipelineId] = P.[PipelineId] INNER JOIN [procfwk].[Orchestrators] D ON P.[OrchestratorId] = D.[OrchestratorId] AND L.[OrchestratorId] = D.[OrchestratorId] WHERE P.[PipelineName] = @PipelineName AND D.[OrchestratorName] = @OrchestratorName AND D.[OrchestratorType] = @OrchestratorType UNION SELECT DISTINCT S.[PrincipalId] AS Id, CAST(DECRYPTBYPASSPHRASE(@OrchestratorName, S.[PrincipalSecret]) AS NVARCHAR(MAX)) AS [Secret] FROM [dbo].[ServicePrincipals] S INNER JOIN [procfwk].[PipelineAuthLink] L ON S.[CredentialId] = L.[CredentialId] INNER JOIN [procfwk].[Orchestrators] D ON L.[OrchestratorId] = D.[OrchestratorId] WHERE D.[OrchestratorName] = @OrchestratorName AND D.[OrchestratorType] = @OrchestratorType ) SELECT TOP 1 @Id = [Id], @Secret = [Secret] FROM cte WHERE [Secret] IS NOT NULL END ELSE IF ([procfwk].[GetPropertyValueInternal]('SPNHandlingMethod')) = 'StoreInKeyVault' BEGIN --get auth details regardless of being pipeline specific and regardless of a pipeline param being passed ;WITH cte AS ( SELECT DISTINCT S.[PrincipalIdUrl] AS Id, S.[PrincipalSecretUrl] AS [Secret] FROM [dbo].[ServicePrincipals] S INNER JOIN [procfwk].[PipelineAuthLink] L ON S.[CredentialId] = L.[CredentialId] INNER JOIN [procfwk].[Pipelines] P ON L.[PipelineId] = P.[PipelineId] INNER JOIN [procfwk].[Orchestrators] D ON P.[OrchestratorId] = D.[OrchestratorId] AND L.[OrchestratorId] = D.[OrchestratorId] WHERE P.[PipelineName] = @PipelineName AND D.[OrchestratorName] = @OrchestratorName AND D.[OrchestratorType] = @OrchestratorType UNION SELECT DISTINCT S.[PrincipalIdUrl] AS Id, S.[PrincipalSecretUrl] AS [Secret] FROM [dbo].[ServicePrincipals] S INNER JOIN [procfwk].[PipelineAuthLink] L ON S.[CredentialId] = L.[CredentialId] INNER JOIN [procfwk].[Orchestrators] D ON L.[OrchestratorId] = D.[OrchestratorId] WHERE D.[OrchestratorName] = @OrchestratorName AND D.[OrchestratorType] = @OrchestratorType ) SELECT TOP 1 @Id = [Id], @Secret = [Secret] FROM cte WHERE [Secret] IS NOT NULL END ELSE BEGIN RAISERROR('Unknown SPN retrieval method.',16,1); RETURN 0; END --return usable values SELECT @Id AS Id, @Secret AS [Secret] END; ================================================ FILE: MetadataDB/procfwkHelpers/Stored Procedures/SetDefaultAlertOutcomes.sql ================================================ CREATE PROCEDURE [procfwkHelpers].[SetDefaultAlertOutcomes] AS BEGIN TRUNCATE TABLE [procfwk].[AlertOutcomes]; INSERT INTO [procfwk].[AlertOutcomes] ( [PipelineOutcomeStatus] ) VALUES ('All'), ('Success'), ('Failed'), ('Unknown'), ('Cancelled'); END; ================================================ FILE: MetadataDB/procfwkHelpers/Stored Procedures/SetDefaultBatchStageLink.sql ================================================ CREATE PROCEDURE [procfwkHelpers].[SetDefaultBatchStageLink] AS BEGIN TRUNCATE TABLE [procfwk].[BatchStageLink] INSERT INTO [procfwk].[BatchStageLink] ( [BatchId], [StageId] ) SELECT b.[BatchId], s.[StageId] FROM [procfwk].[Batches] b INNER JOIN [procfwk].[Stages] s ON s.[StageName] <> 'Speed' WHERE b.[BatchName] = 'Daily' UNION ALL SELECT b.[BatchId], s.[StageId] FROM [procfwk].[Batches] b INNER JOIN [procfwk].[Stages] s ON s.[StageName] = 'Speed' WHERE b.[BatchName] = 'Hourly' END; ================================================ FILE: MetadataDB/procfwkHelpers/Stored Procedures/SetDefaultBatches.sql ================================================ CREATE PROCEDURE [procfwkHelpers].[SetDefaultBatches] AS BEGIN DECLARE @Batches TABLE ( [BatchName] [VARCHAR](225) NOT NULL, [BatchDescription] [VARCHAR](4000) NULL, [Enabled] [BIT] NOT NULL ) INSERT @Batches ( [BatchName], [BatchDescription], [Enabled] ) VALUES ('Daily', N'Daily Worker Pipelines.', 1), ('Hourly', N'Hourly Worker Pipelines.', 1); MERGE INTO [procfwk].[Batches] AS tgt USING @Batches AS src ON tgt.[BatchName] = src.[BatchName] WHEN MATCHED THEN UPDATE SET tgt.[BatchDescription] = src.[BatchDescription], tgt.[Enabled] = src.[Enabled] WHEN NOT MATCHED BY TARGET THEN INSERT ( [BatchName], [BatchDescription], [Enabled] ) VALUES ( src.[BatchName], src.[BatchDescription], src.[Enabled] ) WHEN NOT MATCHED BY SOURCE THEN DELETE; END; ================================================ FILE: MetadataDB/procfwkHelpers/Stored Procedures/SetDefaultOrchestrators.sql ================================================ CREATE PROCEDURE [procfwkHelpers].[SetDefaultOrchestrators] AS BEGIN DECLARE @Orchestrators TABLE ( [OrchestratorName] NVARCHAR(200) NOT NULL, [OrchestratorType] CHAR(3) NOT NULL, [IsFrameworkOrchestrator] BIT NOT NULL, [ResourceGroupName] NVARCHAR(200) NOT NULL, [SubscriptionId] UNIQUEIDENTIFIER NOT NULL, [Description] NVARCHAR(MAX) NULL ) INSERT INTO @Orchestrators ( [OrchestratorName], [OrchestratorType], [IsFrameworkOrchestrator], [Description], [ResourceGroupName], [SubscriptionId] ) VALUES ('FrameworkFactory','ADF',1,'Example Data Factory used for development.','ADF.procfwk','12345678-1234-1234-1234-012345678910'), ('FrameworkFactoryDev','ADF',0,'Example Data Factory used for development deployments.','ADF.procfwk','12345678-1234-1234-1234-012345678910'), ('FrameworkFactoryTest','ADF',0,'Example Data Factory used for testing.','ADF.procfwk','12345678-1234-1234-1234-012345678910'), ('WorkersFactory','ADF',0,'Example Data Factory used to house worker pipelines.','ADF.procfwk','12345678-1234-1234-1234-012345678910'), ('procfwkforsynapse','SYN',0,'Example Synapse instance used to house all pipelines.','ADF.procfwk','12345678-1234-1234-1234-012345678910'); MERGE INTO [procfwk].[Orchestrators] AS tgt USING @Orchestrators AS src ON tgt.[OrchestratorName] = src.[OrchestratorName] AND tgt.[OrchestratorType] = src.[OrchestratorType] WHEN MATCHED THEN UPDATE SET tgt.[IsFrameworkOrchestrator] = src.[IsFrameworkOrchestrator], tgt.[Description] = src.[Description], tgt.[ResourceGroupName] = src.[ResourceGroupName], tgt.[SubscriptionId] = src.[SubscriptionId] WHEN NOT MATCHED BY TARGET THEN INSERT ( [OrchestratorName], [OrchestratorType], [IsFrameworkOrchestrator], [Description], [ResourceGroupName], [SubscriptionId] ) VALUES ( src.[OrchestratorName], src.[OrchestratorType], src.[IsFrameworkOrchestrator], src.[Description], src.[ResourceGroupName], src.[SubscriptionId] ) WHEN NOT MATCHED BY SOURCE THEN DELETE; END; ================================================ FILE: MetadataDB/procfwkHelpers/Stored Procedures/SetDefaultPipelineDependants.sql ================================================ CREATE PROCEDURE [procfwkHelpers].[SetDefaultPipelineDependants] AS BEGIN EXEC [procfwkHelpers].[AddPipelineDependant] @PipelineName = 'Intentional Error', @DependantPipelineName = 'Wait 5'; EXEC [procfwkHelpers].[AddPipelineDependant] @PipelineName = 'Intentional Error', @DependantPipelineName = 'Wait 6'; EXEC [procfwkHelpers].[AddPipelineDependant] @PipelineName = 'Wait 6', @DependantPipelineName = 'Wait 9'; EXEC [procfwkHelpers].[AddPipelineDependant] @PipelineName = 'Wait 9', @DependantPipelineName = 'Wait 10'; END; ================================================ FILE: MetadataDB/procfwkHelpers/Stored Procedures/SetDefaultPipelineParameters.sql ================================================ CREATE PROCEDURE [procfwkHelpers].[SetDefaultPipelineParameters] AS BEGIN DECLARE @PipelineParameters TABLE ( [PipelineId] [INT] NOT NULL, [ParameterName] [VARCHAR](128) NOT NULL, [ParameterValue] [NVARCHAR](MAX) NULL ) INSERT @PipelineParameters ( [PipelineId], [ParameterName], [ParameterValue] ) VALUES (1, 'WaitTime', '3'), (2, 'WaitTime', '6'), (6, 'WaitTime', '9'), (4, 'WaitTime', '5'), (5, 'WaitTime', '2'), (3, 'RaiseErrors', 'false'), (3, 'WaitTime', '10'), (7, 'WaitTime', '3'), (8, 'WaitTime', '5'), (9, 'WaitTime', '7'), (11, 'WaitTime', '10'); MERGE INTO [procfwk].[PipelineParameters] AS tgt USING @PipelineParameters AS src ON tgt.[PipelineId] = src.[PipelineId] AND tgt.[ParameterName] = src.[ParameterName] WHEN MATCHED THEN UPDATE SET tgt.[ParameterValue] = src.[ParameterValue] WHEN NOT MATCHED BY TARGET THEN INSERT ( [PipelineId], [ParameterName], [ParameterValue] ) VALUES ( src.[PipelineId], src.[ParameterName], src.[ParameterValue] ) WHEN NOT MATCHED BY SOURCE THEN DELETE; END; ================================================ FILE: MetadataDB/procfwkHelpers/Stored Procedures/SetDefaultPipelines.sql ================================================ CREATE PROCEDURE [procfwkHelpers].[SetDefaultPipelines] AS BEGIN DECLARE @Pipelines TABLE ( [OrchestratorId] [INT] NOT NULL, [StageId] [INT] NOT NULL, [PipelineName] [NVARCHAR](200) NOT NULL, [LogicalPredecessorId] [INT] NULL, [Enabled] [BIT] NOT NULL ) INSERT @Pipelines ( [OrchestratorId], [StageId], [PipelineName], [LogicalPredecessorId], [Enabled] ) VALUES (1,1 ,'Wait 1' ,NULL ,1), (1,1 ,'Wait 2' ,NULL ,1), (1,1 ,'Intentional Error' ,NULL ,1), (1,1 ,'Wait 3' ,NULL ,1), (1,2 ,'Wait 4' ,NULL ,1), (1,2 ,'Wait 5' ,1 ,1), (1,2 ,'Wait 6' ,1 ,1), (1,2 ,'Wait 7' ,NULL ,1), (1,3 ,'Wait 8' ,1 ,1), (1,3 ,'Wait 9' ,6 ,1), (1,4 ,'Wait 10' ,9 ,1), --speed (1,5 ,'Wait 1' ,NULL ,0), (1,5 ,'Wait 2' ,NULL ,0), (1,5 ,'Wait 3' ,NULL ,0), (1,5 ,'Wait 4' ,NULL ,0), --synapse (5,1 ,'Wait 1' ,NULL ,1), (5,1 ,'Wait 2' ,NULL ,1), (5,1 ,'Wait 3' ,NULL ,1), (5,1 ,'Wait 4' ,NULL ,1); MERGE INTO [procfwk].[Pipelines] AS tgt USING @Pipelines AS src ON tgt.[OrchestratorId] = src.[OrchestratorId] AND tgt.[PipelineName] = src.[PipelineName] AND tgt.[StageId] = src.[StageId] WHEN MATCHED THEN UPDATE SET tgt.[LogicalPredecessorId] = src.[LogicalPredecessorId], tgt.[Enabled] = src.[Enabled] WHEN NOT MATCHED BY TARGET THEN INSERT ( [OrchestratorId], [StageId], [PipelineName], [LogicalPredecessorId], [Enabled] ) VALUES ( src.[OrchestratorId], src.[StageId], src.[PipelineName], src.[LogicalPredecessorId], src.[Enabled] ) WHEN NOT MATCHED BY SOURCE THEN DELETE; END; ================================================ FILE: MetadataDB/procfwkHelpers/Stored Procedures/SetDefaultProperties.sql ================================================ CREATE PROCEDURE [procfwkHelpers].[SetDefaultProperties] AS BEGIN EXEC [procfwkHelpers].[AddProperty] @PropertyName = N'OverideRestart', @PropertyValue = N'0', @Description = N'Should processing not be restarted from the point of failure or should a new execution will be created regardless. 1 = Start New, 0 = Restart. '; EXEC [procfwkHelpers].[AddProperty] @PropertyName = N'PipelineStatusCheckDuration', @PropertyValue = N'30', @Description = N'Duration applied to the Wait activity within the Infant pipeline Until iterations.'; EXEC [procfwkHelpers].[AddProperty] @PropertyName = N'UnknownWorkerResultBlocks', @PropertyValue = N'1', @Description = N'If a worker pipeline returns an unknown status. Should this block and fail downstream pipeline? 1 = Yes, 0 = No.'; EXEC [procfwkHelpers].[AddProperty] @PropertyName = N'CancelledWorkerResultBlocks', @PropertyValue = N'1', @Description = N'If a worker pipeline returns an cancelled status. Should this block and fail downstream pipeline? 1 = Yes, 0 = No.'; EXEC [procfwkHelpers].[AddProperty] @PropertyName = N'UseFrameworkEmailAlerting', @PropertyValue = N'0', @Description = N'Do you want the framework to handle pipeline email alerts via the database metadata? 1 = Yes, 0 = No.'; EXEC [procfwkHelpers].[AddProperty] @PropertyName = N'EmailAlertBodyTemplate', @PropertyValue = N'
Pipeline Name: ##PipelineName###
Status: ##Status###

Execution ID: ##ExecId###
Run ID: ##RunId###

Start Date Time: ##StartDateTime###
End Date Time: ##EndDateTime###
Duration (Minutes): ##Duration###

Called by Orchestrator: ##CalledByOrc###
Executed by Orchestrator Type: ##ExecutedByOrcType###
Executed by Orchestrator: ##ExecutedByOrc###
', @Description = N'Basic HTML template of execution information used as the eventual body in email alerts sent.'; EXEC [procfwkHelpers].[AddProperty] @PropertyName = N'FailureHandling', @PropertyValue = N'Simple', @Description = N'Accepted values: None, Simple, DependencyChain. Controls processing bahaviour in the event of Worker failures. See v1.8 release notes for full details.'; EXEC [procfwkHelpers].[AddProperty] @PropertyName = N'SPNHandlingMethod', @PropertyValue = N'StoreInDatabase', @Description = N'Accepted values: StoreInDatabase, StoreInKeyVault. See v1.8.2 release notes for full details.'; EXEC [procfwkHelpers].[AddProperty] @PropertyName = N'ExecutionPrecursorProc', @PropertyValue = N'[dbo].[ExampleCustomExecutionPrecursor]', @Description = N'This procedure will be called first in the parent pipeline and can be used to perform/update any required custom behaviour in the framework execution. For example, enable/disable Worker pipelines given a certain run time/day. Invalid proc name values will be ignored.' EXEC [procfwkHelpers].[AddProperty] @PropertyName = N'UseExecutionBatches', @PropertyValue = N'0', @Description = N'Establishes if execution batches are used as a level above execution stages within the framework. 1 = True, 0 = False.'; EXEC [procfwkHelpers].[AddProperty] @PropertyName = N'FrameworkFactoryResourceGroup', @PropertyValue = N'ADF.procfwk', @Description = N'Supports various queries where the framework factory is inspecting itself and the resource group cant be inferred.'; EXEC [procfwkHelpers].[AddProperty] @PropertyName = N'PreviousPipelineRunsQueryRange', @PropertyValue = N'-1', @Description = N'Used as a date range, today +- this value, when checking for if an execution for a given pipeline is already running. Must include +- symbol in value.'; END; ================================================ FILE: MetadataDB/procfwkHelpers/Stored Procedures/SetDefaultRecipientPipelineAlerts.sql ================================================ CREATE PROCEDURE [procfwkHelpers].[SetDefaultRecipientPipelineAlerts] AS BEGIN EXEC [procfwkHelpers].[AddRecipientPipelineAlerts] @RecipientName = N'Test User 1', @AlertForStatus = 'All'; EXEC [procfwkHelpers].[AddRecipientPipelineAlerts] @RecipientName = N'Test User 2', @PipelineName = 'Intentional Error', @AlertForStatus = 'Failed'; EXEC [procfwkHelpers].[AddRecipientPipelineAlerts] @RecipientName = N'Test User 3', @PipelineName = 'Wait 1', @AlertForStatus = 'Success, Failed, Cancelled'; END; ================================================ FILE: MetadataDB/procfwkHelpers/Stored Procedures/SetDefaultRecipients.sql ================================================ CREATE PROCEDURE [procfwkHelpers].[SetDefaultRecipients] AS BEGIN DECLARE @Recipients TABLE ( [Name] [VARCHAR](255) NULL, [EmailAddress] [NVARCHAR](500) NOT NULL, [MessagePreference] [CHAR](3) NOT NULL, [Enabled] [BIT] NOT NULL ) INSERT INTO @Recipients ( [Name], [EmailAddress], [MessagePreference], [Enabled] ) VALUES ('Test User 1','test.user1@adfprocfwk.com', 'TO', 1), ('Test User 2','test.user2@adfprocfwk.com', 'CC', 1), ('Test User 3','test.user3@adfprocfwk.com', 'BCC', 1); MERGE INTO [procfwk].[Recipients] AS tgt USING @Recipients AS src ON tgt.[Name] = src.[Name] WHEN MATCHED THEN UPDATE SET tgt.[EmailAddress] = src.[EmailAddress], tgt.[MessagePreference] = src.[MessagePreference], tgt.[Enabled] = src.[Enabled] WHEN NOT MATCHED BY TARGET THEN INSERT ( [Name], [EmailAddress], [MessagePreference], [Enabled] ) VALUES ( src.[Name], src.[EmailAddress], src.[MessagePreference], src.[Enabled] ) WHEN NOT MATCHED BY SOURCE THEN DELETE; END; ================================================ FILE: MetadataDB/procfwkHelpers/Stored Procedures/SetDefaultStages.sql ================================================ CREATE PROCEDURE [procfwkHelpers].[SetDefaultStages] AS BEGIN DECLARE @Stages TABLE ( [StageName] [VARCHAR](225) NOT NULL, [StageDescription] [VARCHAR](4000) NULL, [Enabled] [BIT] NOT NULL ) INSERT @Stages ( [StageName], [StageDescription], [Enabled] ) VALUES ('Extract', N'Ingest all data from source systems.', 1), ('Transform', N'Transform ingested data and apply business logic.', 1), ('Load', N'Load transformed data into data warehouse layer.', 1), ('Serve', N'Load transformed data into semantic layer.', 1), ('Speed', N'Regular loading of frequently used data.', 0); MERGE INTO [procfwk].[Stages] AS tgt USING @Stages AS src ON tgt.[StageName] = src.[StageName] WHEN MATCHED THEN UPDATE SET tgt.[StageDescription] = src.[StageDescription], tgt.[Enabled] = src.[Enabled] WHEN NOT MATCHED BY TARGET THEN INSERT ( [StageName], [StageDescription], [Enabled] ) VALUES ( src.[StageName], src.[StageDescription], src.[Enabled] ) WHEN NOT MATCHED BY SOURCE THEN DELETE; END; ================================================ FILE: MetadataDB/procfwkHelpers/Stored Procedures/SetDefaultSubscription.sql ================================================ CREATE PROCEDURE [procfwkHelpers].[SetDefaultSubscription] AS BEGIN DECLARE @Subscriptions TABLE ( [SubscriptionId] UNIQUEIDENTIFIER NOT NULL, [Name] NVARCHAR(200) NOT NULL, [Description] NVARCHAR(MAX) NULL, [TenantId] UNIQUEIDENTIFIER NOT NULL ) INSERT INTO @Subscriptions ( [SubscriptionId], [Name], [Description], [TenantId] ) VALUES ('12345678-1234-1234-1234-012345678910', 'Default', 'Example value for development environment.', '12345678-1234-1234-1234-012345678910'); MERGE INTO [procfwk].[Subscriptions] AS tgt USING @Subscriptions AS src ON tgt.[SubscriptionId] = src.[SubscriptionId] WHEN MATCHED THEN UPDATE SET tgt.[Name] = src.[Name], tgt.[Description] = src.[Description], tgt.[TenantId] = src.[TenantId] WHEN NOT MATCHED BY TARGET THEN INSERT ( [SubscriptionId], [Name], [Description], [TenantId] ) VALUES ( src.[SubscriptionId], src.[Name], src.[Description], src.[TenantId] ) WHEN NOT MATCHED BY SOURCE THEN DELETE; END; ================================================ FILE: MetadataDB/procfwkHelpers/Stored Procedures/SetDefaultTenant.sql ================================================ CREATE PROCEDURE [procfwkHelpers].[SetDefaultTenant] AS BEGIN DECLARE @Tenants TABLE ( [TenantId] UNIQUEIDENTIFIER NOT NULL, [Name] NVARCHAR(200) NOT NULL, [Description] NVARCHAR(MAX) NULL ) INSERT INTO @Tenants ( [TenantId], [Name], [Description] ) VALUES ('12345678-1234-1234-1234-012345678910', 'Default', 'Example value for development environment.'); MERGE INTO [procfwk].[Tenants] AS tgt USING @Tenants AS src ON tgt.[TenantId] = src.[TenantId] WHEN MATCHED THEN UPDATE SET tgt.[Name] = src.[Name], tgt.[Description] = src.[Description] WHEN NOT MATCHED BY TARGET THEN INSERT ( [TenantId], [Name], [Description] ) VALUES ( src.[TenantId], src.[Name], src.[Description] ) WHEN NOT MATCHED BY SOURCE THEN DELETE; END; ================================================ FILE: MetadataDB/procfwkHelpers/Views/PipelineDependencyChains.sql ================================================ CREATE VIEW [procfwkHelpers].[PipelineDependencyChains] AS SELECT ps.[StageName] AS PredecessorStage, pp.[PipelineName] AS PredecessorPipeline, ds.[StageName] AS DependantStage, dp.[PipelineName] AS DependantPipeline FROM [procfwk].[PipelineDependencies] pd --pipeline dependencies INNER JOIN [procfwk].[Pipelines] pp --predecessor pipelines ON pd.[PipelineId] = pp.[PipelineId] INNER JOIN [procfwk].[Pipelines] dp --dependant pipelines ON pd.[DependantPipelineId] = dp.[PipelineId] INNER JOIN [procfwk].[Stages] ps --predecessor stage ON pp.[StageId] = ps.[StageId] INNER JOIN [procfwk].[Stages] ds --dependant stage ON dp.[StageId] = ds.[StageId]; ================================================ FILE: MetadataDB/procfwkReporting/Views/AverageStageDuration.sql ================================================ CREATE VIEW [procfwkReporting].[AverageStageDuration] AS WITH stageStartEnd AS ( SELECT [LocalExecutionId], [StageId], MIN([StartDateTime]) AS 'StageStart', MAX([EndDateTime]) AS 'StageEnd' FROM [procfwk].[ExecutionLog] GROUP BY [LocalExecutionId], [StageId] ) SELECT s.[StageId], s.[StageName], s.[StageDescription], AVG(DATEDIFF(MINUTE, stageStartEnd.[StageStart], stageStartEnd.[StageEnd])) 'AvgStageRunDurationMinutes' FROM stageStartEnd INNER JOIN [procfwk].[Stages] s ON stageStartEnd.[StageId] = s.[StageId] GROUP BY s.[StageId], s.[StageName], s.[StageDescription] ================================================ FILE: MetadataDB/procfwkReporting/Views/CompleteExecutionErrorLog.sql ================================================ CREATE VIEW [procfwkReporting].[CompleteExecutionErrorLog] AS SELECT exeLog.[LogId] AS ExecutionLogId, errLog.[LogId] AS ErrorLogId, exeLog.[LocalExecutionId], exeLog.[StartDateTime] AS ProcessingDateTime, exeLog.[CallingOrchestratorName], exeLog.[OrchestratorType] AS WorkerOrchestartorType, exeLog.[OrchestratorName] AS WorkerOrchestrator, exeLog.[PipelineName] AS WorkerPipelineName, exeLog.[PipelineStatus], errLog.[ActivityRunId], errLog.[ActivityName], errLog.[ActivityType], errLog.[ErrorCode], errLog.[ErrorType], errLog.[ErrorMessage] FROM [procfwk].[ExecutionLog] exeLog INNER JOIN [procfwk].[ErrorLog] errLog ON exeLog.[LocalExecutionId] = errLog.[LocalExecutionId] AND exeLog.[PipelineRunId] = errLog.[PipelineRunId] INNER JOIN [procfwk].[Stages] stgs ON exeLog.[StageId] = stgs.[StageId] ; ================================================ FILE: MetadataDB/procfwkReporting/Views/CompleteExecutionLog.sql ================================================ CREATE VIEW [procfwkReporting].[CompleteExecutionLog] AS SELECT [LogId], [LocalExecutionId], [StageId], [PipelineId], [CallingOrchestratorName], [ResourceGroupName], [OrchestratorType], [OrchestratorName], [PipelineName], [StartDateTime], [PipelineStatus], [EndDateTime], DATEDIFF(MINUTE, [StartDateTime], [EndDateTime]) 'RunDurationMinutes' FROM [procfwk].[ExecutionLog] ================================================ FILE: MetadataDB/procfwkReporting/Views/CurrentExecutionSummary.sql ================================================ CREATE VIEW [procfwkReporting].[CurrentExecutionSummary] AS SELECT ISNULL([PipelineStatus], 'Not Started') AS 'PipelineStatus', COUNT(0) AS 'RecordCount' FROM [procfwk].[CurrentExecution] GROUP BY [PipelineStatus] ================================================ FILE: MetadataDB/procfwkReporting/Views/LastExecution.sql ================================================ CREATE VIEW [procfwkReporting].[LastExecution] AS WITH maxLog AS ( SELECT MAX([LogId]) AS 'MaxLogId' FROM [procfwk].[ExecutionLog] ), lastExecutionId AS ( SELECT [LocalExecutionId] FROM [procfwk].[ExecutionLog] el1 INNER JOIN maxLog ON maxLog.[MaxLogId] = el1.[LogId] ) SELECT el2.[LogId], el2.[StageId], el2.[PipelineId], el2.[PipelineName], el2.[StartDateTime], el2.[PipelineStatus], el2.[EndDateTime], DATEDIFF(MINUTE, el2.[StartDateTime], el2.[EndDateTime]) AS RunDurationMinutes FROM [procfwk].[ExecutionLog] el2 INNER JOIN lastExecutionId ON el2.[LocalExecutionId] = lastExecutionId.[LocalExecutionId] WHERE el2.[EndDateTime] IS NOT NULL; ================================================ FILE: MetadataDB/procfwkReporting/Views/LastExecutionSummary.sql ================================================ CREATE VIEW [procfwkReporting].[LastExecutionSummary] AS WITH maxLog AS ( SELECT MAX([LogId]) AS 'MaxLogId' FROM [procfwk].[ExecutionLog] ), lastExecutionId AS ( SELECT [LocalExecutionId] FROM [procfwk].[ExecutionLog] el1 INNER JOIN maxLog ON maxLog.[MaxLogId] = el1.[LogId] ) SELECT el2.[LocalExecutionId], DATEDIFF(MINUTE, MIN(el2.[StartDateTime]), MAX(el2.[EndDateTime])) 'RunDurationMinutes' FROM [procfwk].[ExecutionLog] el2 INNER JOIN lastExecutionId ON el2.[LocalExecutionId] = lastExecutionId.[LocalExecutionId] GROUP BY el2.[LocalExecutionId] ================================================ FILE: MetadataDB/procfwkReporting/Views/WorkerParallelismOverTime.sql ================================================ CREATE VIEW [procfwkReporting].[WorkerParallelismOverTime] AS WITH numbers AS ( SELECT TOP 500 ROW_NUMBER() OVER (ORDER BY s1.[object_id]) - 1 AS 'Number' FROM sys.all_columns AS s1 CROSS JOIN sys.all_columns AS s2 ), executionBoundaries AS ( SELECT [LocalExecutionId], CAST(CONVERT(VARCHAR(16), MIN([StartDateTime]), 120) AS DATETIME) AS 'ExecutionStart', CAST(CONVERT(VARCHAR(16), MAX([EndDateTime]), 120) AS DATETIME) AS 'ExecutionEnd' FROM [procfwk].[ExecutionLog] --WHERE -- [LocalExecutionId] = '2BB02783-2A2C-4970-9BEA-0543013BFD5E' GROUP BY [LocalExecutionId] ), wallclockRunning AS ( SELECT CAST(DATEADD(MINUTE, n.[Number], eB.[ExecutionStart]) AS DATE) AS 'WallclockDate', CAST(DATEADD(MINUTE, n.[Number], eB.[ExecutionStart]) AS TIME) AS 'WallclockTime', el.[LocalExecutionId], el.[PipelineId], el.[PipelineName], s.[StageName] FROM executionBoundaries eB CROSS JOIN numbers n INNER JOIN [procfwk].[ExecutionLog] eL ON eB.[LocalExecutionId] = eL.[LocalExecutionId] AND DATEADD(MINUTE, n.[Number], eB.[ExecutionStart]) BETWEEN eL.[StartDateTime] AND eL.[EndDateTime] INNER JOIN [procfwk].[Stages] s ON eL.[StageId] = s.[StageId] ) SELECT [WallclockDate], [WallclockTime], [LocalExecutionId], [StageName], STRING_AGG(ISNULL([PipelineName],' '),', ') As 'PipelineName', COUNT([PipelineId]) AS 'WorkerCount' FROM wallclockRunning GROUP BY [WallclockDate], [WallclockTime], [LocalExecutionId], [StageName] GO ================================================ FILE: MetadataDB/procfwkTesting/Stored Procedures/Add20BatchesFor1000Workers.sql ================================================ CREATE PROCEDURE [procfwkTesting].[Add20BatchesFor1000Workers] AS BEGIN SET NOCOUNT ON; --clear default metadata DELETE FROM [procfwk].[BatchStageLink]; DELETE FROM [procfwk].[Batches]; DELETE FROM [procfwk].[PipelineDependencies]; DBCC CHECKIDENT ('[procfwk].[PipelineDependencies]', RESEED, 0); DELETE FROM [procfwk].[PipelineAlertLink]; DBCC CHECKIDENT ('[procfwk].[PipelineAlertLink]', RESEED, 0); DELETE FROM [procfwk].[Recipients]; DBCC CHECKIDENT ('[procfwk].[Recipients]', RESEED, 0); DELETE FROM [procfwk].[PipelineAuthLink]; DBCC CHECKIDENT ('[procfwk].[PipelineAuthLink]', RESEED, 0); DELETE FROM [dbo].[ServicePrincipals]; DBCC CHECKIDENT ('[dbo].[ServicePrincipals]', RESEED, 0); DELETE FROM [procfwk].[PipelineParameters]; DBCC CHECKIDENT ('[procfwk].[PipelineParameters]', RESEED, 0); DELETE FROM [procfwk].[Pipelines]; DBCC CHECKIDENT ('[procfwk].[Pipelines]', RESEED, 0); --get Orchestrator id DECLARE @OrcId INT SELECT @OrcId = [OrchestratorId] FROM [procfwk].[Orchestrators] WHERE [OrchestratorName] = 'WorkersFactory' AND [OrchestratorType] = 'ADF'; --tweak properties UPDATE [procfwk].[Properties] SET [PropertyValue] = '[dbo].[None]' WHERE [PropertyName] = 'ExecutionPrecursorProc'; UPDATE [procfwk].[Properties] SET [PropertyValue] = '30' WHERE [PropertyName] = 'PipelineStatusCheckDuration'; UPDATE [procfwk].[Properties] SET [PropertyValue] = '0' WHERE [PropertyName] = 'UseFrameworkEmailAlerting'; UPDATE [procfwk].[Properties] SET [PropertyValue] = '1' WHERE [PropertyName] = 'UseExecutionBatches'; --insert 200 pipelines ;WITH cte AS ( SELECT TOP 200 ROW_NUMBER() OVER (ORDER BY s1.[object_id]) AS Number FROM sys.all_columns AS s1 CROSS JOIN sys.all_columns AS s2 ) INSERT INTO [procfwk].[Pipelines] ( [OrchestratorId], [StageId], [PipelineName], [LogicalPredecessorId], [Enabled] ) SELECT @OrcId, CASE WHEN [Number] <= 50 THEN 1 WHEN [Number] > 50 AND [Number] <= 100 THEN 2 WHEN [Number] > 100 AND [Number] <= 150 THEN 3 WHEN [Number] > 150 AND [Number] <= 200 THEN 4 END, 'Wait ' + CAST([Number] AS VARCHAR), NULL, 1 FROM cte; --disable other execution stages if exist UPDATE [procfwk].[Stages] SET [Enabled] = 1; UPDATE [procfwk].[Stages] SET [Enabled] = 0 WHERE [StageId] > 4; --insert 200 pipeline parameters INSERT INTO [procfwk].[PipelineParameters] ( [PipelineId], [ParameterName], [ParameterValue] ) SELECT [PipelineId], 'WaitTime', LEFT(ABS(CAST(CAST(NEWID() AS VARBINARY) AS INT)),1) FROM [procfwk].[Pipelines]; --insert batches ;WITH batchNames AS ( SELECT 'One' AS [Name] UNION SELECT 'Two' UNION SELECT 'Three' UNION SELECT 'Four' UNION SELECT 'Five' UNION SELECT 'Six' UNION SELECT 'Seven' UNION SELECT 'Eight' UNION SELECT 'Nine' UNION SELECT 'Ten' UNION SELECT 'Eleven' UNION SELECT 'Twelve' UNION SELECT 'Thirteen' UNION SELECT 'Fourteen' UNION SELECT 'Fifteen' UNION SELECT 'Sixteen' UNION SELECT 'Seventeen' UNION SELECT 'Eighteen' UNION SELECT 'Nineteen' UNION SELECT 'Twenty' ) INSERT INTO [procfwk].[Batches] ( [BatchName], [BatchDescription], [Enabled] ) SELECT [Name], '1000 Workers Test', 1 FROM batchNames --allocation batches to stages evenly ;WITH maxStages AS ( SELECT MAX([StageId]) AS Id FROM [procfwk].[Stages] WHERE [Enabled] = 1 ) INSERT INTO [procfwk].[BatchStageLink] SELECT b.[BatchId], CASE WHEN (ROW_NUMBER() OVER (ORDER BY b.[BatchName] DESC) * 1) % maxStages.[Id] = 0 THEN maxStages.[Id] ELSE (ROW_NUMBER() OVER (ORDER BY b.[BatchName] DESC) * 1) % maxStages.[Id] END AS Stage FROM [procfwk].[Batches] b CROSS JOIN maxStages; /* --generate the c# for the NUnit test: SELECT [BatchName], 'private GrandparentHelper _helperBatch' + [BatchName] + ';', '_helperBatch' + [BatchName] + ' = new GrandparentHelper().WithParameter("BatchName", "' + [BatchName] + '");', 'var batch' + [BatchName] + ' = _helperBatch' + [BatchName] + '.RunPipeline();', 'batch' + [BatchName] + ',', '[Test]' + CHAR(13) + 'public void Then' + [BatchName] + 'BatchPipelineOutcomeIsSucceeded()' + CHAR(13) + '{' + CHAR(13) + ' _helperBatch' + [BatchName] + '.RunOutcome.Should().Be("Succeeded");' + CHAR(13) + '}' FROM [procfwk].[Batches] */ END; ================================================ FILE: MetadataDB/procfwkTesting/Stored Procedures/Add300WorkerPipelineBatches.sql ================================================ CREATE PROCEDURE [procfwkTesting].[Add300WorkerPipelineBatches] AS BEGIN SET NOCOUNT ON; --clear default metadata DELETE FROM [procfwk].[BatchStageLink]; DELETE FROM [procfwk].[Batches]; --add batch details ;WITH sourceData AS ( SELECT '0to300' AS BatchName, 'The first 300.' AS BatchDescription, 1 AS [Enabled] UNION SELECT '301to600', 'The second 300.', 1 ) MERGE INTO [procfwk].[Batches] AS tgt USING sourceData AS src ON tgt.[BatchName] = src.[BatchName] WHEN MATCHED THEN UPDATE SET tgt.[BatchDescription] = src.[BatchDescription], tgt.[Enabled] = src.[Enabled] WHEN NOT MATCHED BY TARGET THEN INSERT ( [BatchName], [BatchDescription], [Enabled] ) VALUES ( src.[BatchName], src.[BatchDescription], src.[Enabled] ) WHEN NOT MATCHED BY SOURCE THEN DELETE; --link batches to stages INSERT INTO [procfwk].[BatchStageLink] ( [BatchId], [StageId] ) SELECT b.[BatchId], s.[StageId] FROM [procfwk].[Batches] b INNER JOIN [procfwk].[Stages] s ON s.[Enabled] = 1; END; ================================================ FILE: MetadataDB/procfwkTesting/Stored Procedures/Add300WorkerPipelines.sql ================================================ CREATE PROCEDURE [procfwkTesting].[Add300WorkerPipelines] AS BEGIN SET NOCOUNT ON; --clear default metadata DELETE FROM [procfwk].[PipelineDependencies]; DBCC CHECKIDENT ('[procfwk].[PipelineDependencies]', RESEED, 0); DELETE FROM [procfwk].[PipelineAlertLink]; DBCC CHECKIDENT ('[procfwk].[PipelineAlertLink]', RESEED, 0); DELETE FROM [procfwk].[Recipients]; DBCC CHECKIDENT ('[procfwk].[Recipients]', RESEED, 0); DELETE FROM [procfwk].[PipelineAuthLink]; DBCC CHECKIDENT ('[procfwk].[PipelineAuthLink]', RESEED, 0); DELETE FROM [dbo].[ServicePrincipals]; DBCC CHECKIDENT ('[dbo].[ServicePrincipals]', RESEED, 0); DELETE FROM [procfwk].[PipelineParameters]; DBCC CHECKIDENT ('[procfwk].[PipelineParameters]', RESEED, 0); DELETE FROM [procfwk].[Pipelines]; DBCC CHECKIDENT ('[procfwk].[Pipelines]', RESEED, 0); --get Orchestrator id DECLARE @OrcId INT SELECT @OrcId = [OrchestratorId] FROM [procfwk].[Orchestrators] WHERE [OrchestratorName] = 'WorkersFactory' AND [OrchestratorType] = 'ADF'; --insert 300 pipelines ;WITH cte AS ( SELECT TOP 300 ROW_NUMBER() OVER (ORDER BY s1.[object_id]) AS Number FROM sys.all_columns AS s1 CROSS JOIN sys.all_columns AS s2 ) INSERT INTO [procfwk].[Pipelines] ( [OrchestratorId], [StageId], [PipelineName], [LogicalPredecessorId], [Enabled] ) SELECT @OrcId, CASE WHEN [Number] <= 100 THEN 1 WHEN [Number] > 100 AND [Number] <= 200 THEN 2 WHEN [Number] > 200 AND [Number] <= 300 THEN 3 END, 'Wait ' + CAST([Number] AS VARCHAR), NULL, 1 FROM cte; --disable other execution stages if exist UPDATE [procfwk].[Stages] SET [Enabled] = 0 WHERE [StageId] > 3; --insert 300 pipeline parameters INSERT INTO [procfwk].[PipelineParameters] ( [PipelineId], [ParameterName], [ParameterValue] ) SELECT [PipelineId], 'WaitTime', LEFT(ABS(CAST(CAST(NEWID() AS VARBINARY) AS INT)),2) FROM [procfwk].[Pipelines]; END; ================================================ FILE: MetadataDB/procfwkTesting/Stored Procedures/CleanUpMetadata.sql ================================================ CREATE PROCEDURE [procfwkTesting].[CleanUpMetadata] AS BEGIN EXEC [procfwkHelpers].[DeleteMetadataWithIntegrity]; EXEC [procfwkHelpers].[DeleteMetadataWithoutIntegrity]; END; ================================================ FILE: MetadataDB/procfwkTesting/Stored Procedures/GetRunIdWhenAvailable.sql ================================================ CREATE PROCEDURE [procfwkTesting].[GetRunIdWhenAvailable] ( @PipelineName NVARCHAR(200) = NULL ) AS BEGIN IF @PipelineName IS NULL BEGIN WHILE 1=1 BEGIN IF EXISTS ( SELECT TOP 1 [PipelineRunId] FROM [procfwk].[CurrentExecution] WHERE [PipelineRunId] IS NOT NULL ) BEGIN BREAK; END WAITFOR DELAY '00:00:10'; END; SELECT TOP 1 CAST([PipelineRunId] AS VARCHAR(36)) AS RunId FROM [procfwk].[CurrentExecution] WHERE [PipelineRunId] IS NOT NULL END ELSE IF @PipelineName IS NOT NULL BEGIN WHILE 1=1 BEGIN IF EXISTS ( SELECT TOP 1 [PipelineRunId] FROM [procfwk].[CurrentExecution] WHERE [PipelineRunId] IS NOT NULL AND [PipelineName] = @PipelineName ) BEGIN BREAK; END WAITFOR DELAY '00:00:10'; END; SELECT TOP 1 CAST([PipelineRunId] AS VARCHAR(36)) AS RunId FROM [procfwk].[CurrentExecution] WHERE [PipelineRunId] IS NOT NULL AND [PipelineName] = @PipelineName END ELSE BEGIN RAISERROR('Unknown use of testing procedure.',16,1); END END; ================================================ FILE: MetadataDB/procfwkTesting/Stored Procedures/ResetMetadata.sql ================================================ CREATE PROCEDURE [procfwkTesting].[ResetMetadata] AS BEGIN EXEC [procfwkHelpers].[SetDefaultProperties]; EXEC [procfwkHelpers].[SetDefaultTenant]; EXEC [procfwkHelpers].[SetDefaultSubscription]; EXEC [procfwkHelpers].[SetDefaultOrchestrators]; EXEC [procfwkHelpers].[SetDefaultBatches]; EXEC [procfwkHelpers].[SetDefaultStages]; EXEC [procfwkHelpers].[SetDefaultBatchStageLink]; EXEC [procfwkHelpers].[SetDefaultPipelines]; EXEC [procfwkHelpers].[SetDefaultPipelineParameters]; EXEC [procfwkHelpers].[SetDefaultPipelineDependants]; EXEC [procfwkHelpers].[SetDefaultRecipients]; EXEC [procfwkHelpers].[SetDefaultAlertOutcomes]; EXEC [procfwkHelpers].[SetDefaultRecipientPipelineAlerts]; END; ================================================ FILE: MetadataDBTests/MetadataDBTests.sqlproj ================================================  Debug AnyCPU MetadataDBTests 2.0 4.1 {32d54b90-932d-44a3-911d-007c4b90bc55} Microsoft.Data.Tools.Schema.Sql.SqlAzureV12DatabaseSchemaProvider Database MetadataDBTests MetadataDBTests 1033, CI BySchemaAndSchemaType True v4.7.2 CS Properties False True True bin\Release\ $(MSBuildProjectName).sql False pdbonly true false true prompt 4 bin\Debug\ $(MSBuildProjectName).sql false true full false true true prompt 4 11.0 True 11.0 MetadataDB {202ebf84-a56b-4999-92a3-10f7ffe4ef25} True False $(DacPacRootPath)\Extensions\Microsoft\SQLDB\Extensions\SqlServer\AzureV12\SqlSchemas\master.dacpac False master tSQLt.dacpac False ================================================ FILE: MetadataDBTests/Scripts/Script.PostDeployment.sql ================================================ /* Post-Deployment Script Template -------------------------------------------------------------------------------------- This file contains SQL statements that will be appended to the build script. Use SQLCMD syntax to include a file in the post-deployment script. Example: :r .\myfile.sql Use SQLCMD syntax to reference a variable in the post-deployment script. Example: :setvar TableName MyTable SELECT * FROM [$(TableName)] -------------------------------------------------------------------------------------- */ :r ..\..\MetadataDB\Scripts\Script.PostDeployment.sql ================================================ FILE: MetadataDBTests/Scripts/Script.PreDeployment.sql ================================================ /* Pre-Deployment Script Template -------------------------------------------------------------------------------------- This file contains SQL statements that will be executed before the build script. Use SQLCMD syntax to include a file in the pre-deployment script. Example: :r .\myfile.sql Use SQLCMD syntax to reference a variable in the pre-deployment script. Example: :setvar TableName MyTable SELECT * FROM [$(TableName)] -------------------------------------------------------------------------------------- */ :r ..\..\MetadataDB\Scripts\Script.PreDeployment.sql ================================================ FILE: MetadataDBTests/_TestClasses/procfwk_GetPropertyValue.sql ================================================ CREATE SCHEMA procfwk_GetPropertyValue AUTHORIZATION [dbo]; GO EXEC sp_addextendedproperty @name = N'tSQLt.TestClass', @value = 1, @level0type = N'SCHEMA', @level0name = N'procfwk_GetPropertyValue' ================================================ FILE: MetadataDBTests/_TestClasses/procfwk_GetPropertyValueInternal.sql ================================================ CREATE SCHEMA procfwk_GetPropertyValueInternal AUTHORIZATION [dbo]; GO EXEC sp_addextendedproperty @name = N'tSQLt.TestClass', @value = 1, @level0type = N'SCHEMA', @level0name = N'procfwk_GetPropertyValueInternal' ================================================ FILE: MetadataDBTests/procfwk_GetPropertyValue/setup.sql ================================================ CREATE PROCEDURE procfwk_GetPropertyValue.[setup] AS DECLARE @propertyName NVARCHAR(128) = 'TestProperty' EXEC tSQLt.FakeTable 'procfwk.CurrentProperties' INSERT INTO procfwk.CurrentProperties ( [PropertyName] , [PropertyValue] ) VALUES ( @propertyName , 'TestPropertyValue' ) ================================================ FILE: MetadataDBTests/procfwk_GetPropertyValue/test WHEN property does not exist THEN error raised.sql ================================================ CREATE PROCEDURE procfwk_GetPropertyValue.[test WHEN property does not exist THEN error raised] AS -- ARRANGE DECLARE @propertyName NVARCHAR(128) = 'OtherProperty' EXEC tSQLt.FakeTable 'procfwk.Properties' INSERT INTO procfwk.Properties ( [PropertyName] , [PropertyValue] ) VALUES ( 'TestProperty' , 'TestPropertyValue' ) -- EXPECT EXEC tSQLt.ExpectException @ExpectedMessagePattern = '%Invalid property name %' -- ACT CREATE TABLE #actual ( PropertyValue NVARCHAR(4000) ) INSERT INTO #actual ( PropertyValue ) EXEC procfwk.GetPropertyValue @propertyName ================================================ FILE: MetadataDBTests/procfwk_GetPropertyValue/test WHEN property exists THEN property value returned.sql ================================================ CREATE PROCEDURE procfwk_GetPropertyValue.[test WHEN property exists THEN property value returned] AS -- ARRANGE DECLARE @propertyName NVARCHAR(128) = 'TestProperty' EXEC tSQLt.FakeTable 'procfwk.Properties' INSERT INTO procfwk.Properties ( [PropertyName] , [PropertyValue] ) VALUES ( 'TestProperty' , 'TestPropertyValue' ) SELECT N'TestPropertyValue' AS PropertyValue INTO #expected -- ACT CREATE TABLE #actual ( PropertyValue NVARCHAR(4000) ) INSERT INTO #actual ( PropertyValue ) EXEC procfwk.GetPropertyValue @propertyName -- ASSERT EXEC tSQLt.AssertEqualsTable @Expected = '#expected' , @Actual = '#actual' ================================================ FILE: MetadataDBTests/procfwk_GetPropertyValue/test WHEN property invalidated THEN error raised.sql ================================================ CREATE PROCEDURE procfwk_GetPropertyValue.[test WHEN property invalidated THEN error raised] AS -- ARRANGE DECLARE @propertyName NVARCHAR(128) = 'TestProperty' EXEC tSQLt.FakeTable 'procfwk.Properties' INSERT INTO procfwk.Properties ( [PropertyName] , [PropertyValue] , ValidTo ) VALUES ( 'TestProperty' , 'TestPropertyValue' , GETDATE() ) -- EXPECT EXEC tSQLt.ExpectException @ExpectedMessagePattern = '% does not have a current valid version %' -- ACT CREATE TABLE #actual ( PropertyValue NVARCHAR(4000) ) INSERT INTO #actual ( PropertyValue ) EXEC procfwk.GetPropertyValue @propertyName ================================================ FILE: MetadataDBTests/procfwk_GetPropertyValue/test WHEN property name is null THEN error raised.sql ================================================ CREATE PROCEDURE procfwk_GetPropertyValue.[test WHEN property name is null THEN error raised] AS -- ARRANGE DECLARE @propertyName NVARCHAR(128) = NULL EXEC tSQLt.FakeTable 'procfwk.Properties' INSERT INTO procfwk.Properties ( [PropertyName] , [PropertyValue] ) VALUES ( 'TestProperty' , 'TestPropertyValue' ) -- EXPECT EXEC tSQLt.ExpectException @ExpectedMessagePattern = '%Invalid property name %' -- ACT CREATE TABLE #actual ( PropertyValue NVARCHAR(4000) ) INSERT INTO #actual ( PropertyValue ) EXEC procfwk.GetPropertyValue @propertyName ================================================ FILE: MetadataDBTests/procfwk_GetPropertyValueInternal/test WHEN property does not exist THEN empty string returned.sql ================================================ CREATE PROCEDURE procfwk_GetPropertyValueInternal.[test WHEN property name does not exist THEN empty string returned] AS -- ARRANGE DECLARE @propertyName NVARCHAR(128) = 'TestProperty' EXEC tSQLt.FakeTable 'procfwk.CurrentProperties' DECLARE @expected NVARCHAR(4000) = '' -- ACT DECLARE @actual NVARCHAR(4000) = procfwk.GetPropertyValueInternal(@propertyName) -- ASSERT EXEC tSQLt.AssertEquals @Expected = @expected , @Actual = @actual ================================================ FILE: MetadataDBTests/procfwk_GetPropertyValueInternal/test WHEN property exists THEN property value returned.sql ================================================ CREATE PROCEDURE procfwk_GetPropertyValueInternal.[test WHEN property exists THEN property value returned] AS -- ARRANGE DECLARE @propertyName NVARCHAR(128) = 'TestProperty' EXEC tSQLt.FakeTable 'procfwk.CurrentProperties' INSERT INTO procfwk.CurrentProperties ( [PropertyName] , [PropertyValue] ) VALUES ( 'TestProperty' , 'TestPropertyValue' ) DECLARE @expected NVARCHAR(4000) = 'TestPropertyValue' -- ACT DECLARE @actual NVARCHAR(4000) = procfwk.GetPropertyValueInternal(@propertyName) -- ASSERT EXEC tSQLt.AssertEquals @Expected = @expected , @Actual = @actual ================================================ FILE: MetadataDBTests/procfwk_GetPropertyValueInternal/test WHEN property name is null THEN empty string returned.sql ================================================ CREATE PROCEDURE procfwk_GetPropertyValueInternal.[test WHEN property name is null THEN empty string returned] AS -- ARRANGE DECLARE @propertyName NVARCHAR(128) = NULL EXEC tSQLt.FakeTable 'procfwk.CurrentProperties' INSERT INTO procfwk.CurrentProperties ( [PropertyName] , [PropertyValue] ) VALUES ( 'TestProperty' , 'TestPropertyValue' ) DECLARE @expected NVARCHAR(4000) = '' -- ACT DECLARE @actual NVARCHAR(4000) = procfwk.GetPropertyValueInternal(@propertyName) -- ASSERT EXEC tSQLt.AssertEquals @Expected = @expected , @Actual = @actual ================================================ FILE: Notebooks/Databricks - Throw Exception.scala ================================================ // Databricks notebook source import scala.util.Try dbutils.widgets.text("RaiseError", "","") // COMMAND ---------- val raiseError = Try(dbutils.widgets.get("RaiseError").toBoolean).getOrElse(false) if(raiseError) { throw new Exception("The Notebook intentionally failed.") } ================================================ FILE: Notebooks/Metadata Guide and Handy Code Snippets.ipynb ================================================ { "metadata": { "kernelspec": { "name": "SQL", "display_name": "SQL", "language": "sql" }, "language_info": { "name": "sql", "version": "" } }, "nbformat_minor": 2, "nbformat": 4, "cells": [ { "cell_type": "markdown", "source": [ "# Welcome to the ADF.procfwk Handy User Guide Notebook\r\n", "\r\n", "This Notebook provides a series of short code snippets and narrative to help developers work with processing framework metadata.\r\n", "\r\n", "![alt text](https://mrpaulandrew.files.wordpress.com/2020/03/adfprocfwk-icon.png \"ADF.procfwk Icon\")\r\n", "\r\n", "Note: do not just run all cells in this Notebook. This may make unwanted changes to your deployed solution. The EXEC code is written with example values for you to update." ], "metadata": { "azdata_cell_guid": "be267c69-3b61-4daf-980e-0adf85f85108" } }, { "cell_type": "code", "source": [ "SET NOCOUNT ON;" ], "metadata": { "azdata_cell_guid": "4b561a97-5508-4001-9bb3-4d2ed7fe3a2f" }, "outputs": [], "execution_count": null }, { "cell_type": "markdown", "source": [ "***\r\n", "# Getting Current & Previous Execution Details\r\n", "\r\n", "* The [CurrentExecution] table only contains data for processing runs that our in progress or in an incomplete state.\r\n", "\r\n", "* The [ExecutionLog] table and [CompleteExecutionLog] views contains all other log details for execution runs." ], "metadata": { "azdata_cell_guid": "1f574591-3c81-4fa4-ad6a-54a06aa75e2b" } }, { "cell_type": "code", "source": [ "SELECT * FROM [procfwk].[CurrentExecution];\r\n", "\r\n", "SELECT * FROM [procfwk].[CompleteExecutionLog];\r\n", "\r\n", "SELECT * FROM [procfwk].[LastExecution];\r\n", "\r\n", "SELECT TOP 100 * FROM [procfwk].[ExecutionLog] ORDER BY [LocalExecutionId], [StageId], [PipelineId], [StartDateTime];\r\n", "\r\n", "EXEC [procfwk].[GetExecutionDetails];" ], "metadata": { "azdata_cell_guid": "ad517e7c-9517-404b-9e4d-c58e4b7dbd0f" }, "outputs": [], "execution_count": null }, { "cell_type": "markdown", "source": [ "***\r\n", "# Getting Error Details\r\n", "\r\n", "* The [CurrentExecution] table and the [CompleteExecutionErrorLog] view contain details of error logged for failed pipeline activities. A single pipeline can contain multiple activities that if executed in parallel may generate multiple errrors for a single pipeline Run ID." ], "metadata": { "azdata_cell_guid": "f6362034-9ff7-4a6d-a39f-789787d445df" } }, { "cell_type": "code", "source": [ "SELECT * FROM [procfwk].[ErrorLog];\r\n", "\r\n", "SELECT * FROM [procfwk].[CompleteExecutionErrorLog];\r\n", "\r\n", "EXEC [procfwk].[GetExecutionDetails];" ], "metadata": { "azdata_cell_guid": "5a3106a2-c3c8-4985-923e-79410022ff90" }, "outputs": [], "execution_count": null }, { "cell_type": "markdown", "source": [ "***\r\n", "# Reviewing Basic Processing Metadata\r\n", "The following tables have been ordered as per there level within the processing framework. A logical hierarchy exists between these tables and is enforced via database constraints connecting primary and foreign keys.\r\n", "* Data Factory's\r\n", "* Stages\r\n", "* Pipelines\r\n", "* Pipeline Parameters" ], "metadata": { "azdata_cell_guid": "88506809-f0a5-422d-bf7f-6e6e5082a8d0" } }, { "cell_type": "code", "source": [ "SELECT * FROM [procfwk].[DataFactorys];\r\n", "\r\n", "SELECT * FROM [procfwk].[Stages];\r\n", "\r\n", "SELECT * FROM [procfwk].[Pipelines];\r\n", "\r\n", "SELECT * FROM [procfwk].[PipelineParameters];" ], "metadata": { "azdata_cell_guid": "1a77b2ec-ffa9-4d67-91db-a1fb3acb1693" }, "outputs": [], "execution_count": null }, { "cell_type": "markdown", "source": [ "***\r\n", "# Review and Check Logical Pipeline Metadata Integrity\r\n", "\r\n", "The following procedure uses the optional attribute [LogicalPredecessorId] within the table [procfwk].[Pipelines] to create a chain of dependencies between processes. This chain has zero affect of the execution of the framework, but does allow data lineage chains to be reviewed and created without impacting processing.\r\n", "\r\n", "The procedure also implements a series of case statement checks against pipeline chains to advise where metadata issues may exist. For example:\r\n", "\r\n", "* Pipeline could be moved to an earlier stage if it has no predecessors and/or isn't in an earlier stage.\r\n", "* Dependency issue, predeccessor pipeline is currently running in the same stage as successor.\r\n", "* Disabled pipeline has downstream successors.\r\n", "* Disabled stage has downstream successors.\r\n", "\r\n", "It is recommended that these advisory points are reviewed and the logical chain of pipelines updated to inform better framework execution.\r\n", "\r\n", "" ], "metadata": { "azdata_cell_guid": "8fae870b-a02f-4b4b-8122-bd94e2fbd39b" } }, { "cell_type": "code", "source": [ "EXEC [procfwk].[CheckStageAndPiplineIntegrity];" ], "metadata": { "azdata_cell_guid": "eeded0ba-002d-4c6f-b4ff-1047aa38ee62" }, "outputs": [], "execution_count": null }, { "cell_type": "markdown", "source": [ "***\r\n", "# Checking and Adding Framework Properties\r\n", "* Use the [CurrentProperties] view to check only the latest version of property values.\r\n", "* Use the complete [Properties] table to see all versions of framework properties with valid from and to dates.\r\n", "* Use the stored procedure [AddProperty] to add a new property/value or update an existing property/value.\r\n", "* Use the stored procedure [GetPropertyValue] to get the latest version of a single property value." ], "metadata": { "azdata_cell_guid": "ae575f21-e47d-4544-96c5-44e1af85c9d9" } }, { "cell_type": "code", "source": [ "SELECT * FROM [procfwk].[CurrentProperties];\r\n", "\r\n", "SELECT * FROM [procfwk].[Properties];\r\n", "\r\n", "EXEC [procfwk].[AddProperty] \r\n", "\t@PropertyName = 'TenantId',\r\n", "\t@PropertyValue = '1234-1234-1234-1234-1234',\r\n", "\t@Description = 'Used to provide authentication throughout the framework execution.';\r\n", "\r\n", "EXEC [procfwk].[GetPropertyValue]\r\n", "\t@PropertyName = N'TenantId';" ], "metadata": { "azdata_cell_guid": "e0e57080-aa45-49d6-a994-7fb83dfe4a08" }, "outputs": [], "execution_count": null }, { "cell_type": "markdown", "source": [ "***\r\n", "# Checking, Getting, Adding and Deleting Service Principals\r\n", "\r\n", "To **check** which SPN details are being used follow the guidance in the framework for pipeline authentication:\r\n", "\r\n", "* The [ServicePrincipals] table in the [dbo] schema details which credentials are being used by the framework. These details are specific to an Azure Tenant. However, this table also needs to be joined with table [procfwk].[PipelineAuthLink] to better understand which credentials are being used by which Data Factory/Pipeline.\r\n", "\r\n", "To **get** actual unencrypted SPN details the stored procedure [procfwk].[GetServicePrincipal].\r\n", "\r\n", "Be careful not to change the Tenant ID property after Service Principals have been added.\r\n", "\r\n", "To **add** Service Principals use the stored procedure [procfwk].[AddServicePrincipal]. This will handle the creation of the links between Data Factory's and Pipelines.\r\n", "\r\n", "To **delete** Service Principals use the stored procedure [procfwk].[DeleteServicePrincipal]. This will also handle the removal of the links between Data Factory's and Pipelines.\r\n", "\r\n", "Also, be aware that as part of the new metadata integrity checks introduced in v1.3 of the framework. This will ensure all enabled pipelines have a valid SPN before the execution run starts." ], "metadata": { "azdata_cell_guid": "86d98611-0f43-4c65-bf91-ea8d3542e705" } }, { "cell_type": "code", "source": [ "--Checking:\r\n", "SELECT * FROM [dbo].[ServicePrincipals];\r\n", "\r\n", "SELECT\r\n", "\tSP.[PrincipalName],\r\n", "\tDF.[ResourceGroupName],\r\n", "\tDF.[DataFactoryName],\r\n", "\tPP.[PipelineName]\r\n", "FROM\r\n", "\t[procfwk].[PipelineAuthLink] AL\r\n", "\tINNER JOIN [procfwk].[DataFactorys] DF\r\n", "\t\tON AL.[DataFactoryId] = DF.[DataFactoryId]\r\n", "\tINNER JOIN [procfwk].[Pipelines] PP\r\n", "\t\tON AL.[PipelineId] = PP.[PipelineId]\r\n", "\tINNER JOIN [dbo].[ServicePrincipals] SP\r\n", "\t\tON AL.[CredentialId] = SP.[CredentialId];\r\n", "\r\n", "--Getting:\r\n", "EXEC [procfwk].[GetServicePrincipal]\r\n", "\t@DataFactory = 'FrameworkFactory',\r\n", "\t@PipelineName = 'Wait 1';\r\n", "\r\n", "--Adding:\r\n", "EXEC [procfwk].[AddServicePrincipal]\r\n", "\t@DataFactory = N'FrameworkFactory',\r\n", "\t@PrincipalId = N'1234-1234-1234-1234-1234',\r\n", "\t@PrincipalSecret = N'Passw0rd123!',\r\n", "\t@PrincipalName = N'ADFFrameworkExecutor',\r\n", "\t@SpecificPipelineName = N'Wait 1' --Optional parameter\r\n", "\r\n", "--Deleteing:\r\n", "EXEC [procfwk].[DeleteServicePrincipal]\r\n", "\t@DataFactory = N'FrameworkFactory',\r\n", "\t@PrincipalId = N'1234-1234-1234-1234-1234',\r\n", "\t@SpecificPipelineName = N'Wait 1' --Optional parameter" ], "metadata": { "azdata_cell_guid": "ac0c404d-5a8a-4c39-96ff-66e198fffd9f" }, "outputs": [], "execution_count": null }, { "cell_type": "markdown", "source": [ "***\r\n", "# Pre Execution Metadata Integrity Checks\r\n", "\r\n", "This procedure now acts as a pass/fail check before a given framework exeuction runs starts. If any of the checks fail the Data Factory pipelines will not start and the execution run will be stalled.\r\n", "\r\n", "In debug mode details of the checks that failed can be reviewed and fixed.\r\n", "" ], "metadata": { "azdata_cell_guid": "6698a16a-026d-4ab0-bae0-5a3b0a672649" } }, { "cell_type": "code", "source": [ "EXEC [procfwk].[CheckMetadataIntegrity]\r\n", "\t@DebugMode = 1" ], "metadata": { "azdata_cell_guid": "f4247341-87a7-46d3-b55e-1371a48f6d81" }, "outputs": [], "execution_count": null }, { "cell_type": "markdown", "source": [ "***\r\n", "# Email Alerting\r\n", "\r\n", "The following snippets provide examples on setting up alerting and adding users to the metadata.\r\n", "\r\n", "Note: SMTP details need to be provided in the Application Settings of the Azure Function App for email to be sent.\r\n", "" ], "metadata": { "azdata_cell_guid": "f0e2fc3b-b6d9-49be-874f-d8fd59d8c7bc" } }, { "cell_type": "code", "source": [ "--Is framework email alerting enabled?\r\n", "SELECT CASE [PropertyValue] WHEN 1 THEN 'Yes' WHEN 0 THEN 'No' ELSE 'Unknown' END AS AlertingStatus FROM [procfwk].[CurrentProperties] WHERE [PropertyName] = 'UseFrameworkEmailAlerting';\r\n", "\r\n", "--Enable framework alerting:\r\n", "EXEC [procfwk].[AddProperty]\r\n", "\t@PropertyName = N'UseFrameworkEmailAlerting',\r\n", "\t@PropertyValue = N'1',\r\n", "\t@Description = N'Do you want the framework to handle pipeline email alerts via the database metadata? 1 = Yes, 0 = No.';\r\n", "\r\n", "--List current recipients:\r\n", "SELECT [RecipientId], [Name], [EmailAddress], [MessagePreference], [Enabled] FROM [procfwk].[Recipients];\r\n", "\r\n", "--Add a new recipient:\r\n", "INSERT INTO [procfwk].[Recipients] VALUES ('Test User 3','test.user3@adfprocfwk.com', 'BCC', 1);\r\n", "\r\n", "--Add an alert link for a recipient and pipeline:\r\n", "EXEC [procfwk].[AddRecipientPipelineAlerts]\r\n", "\t@RecipientName = N'Test User 3',\r\n", " @PipelineName = 'Wait 1', --optional, default = all\r\n", "\t@AlertForStatus = 'Success, Failed, Cancelled'; --optional, default = all\r\n", "\r\n", "--Remove recipient alerts:\r\n", "EXEC [procfwk].[DeleteRecipientAlerts]\r\n", "\t@EmailAddress = N'test.user1@adfprocfwk.com',\r\n", "\t@SoftDeleteOnly = 1; --soft delete sets Enabled = 0\r\n", "\r\n", "--Check a recipients alert subscriptions and preferences for all pipelines:\r\n", "SELECT\r\n", "\tr.[Name] AS RecipientName,\r\n", "\tp.[PipelineName],\r\n", "\tr.[MessagePreference],\r\n", "\tao.[PipelineOutcomeStatus],\r\n", "\tp.[Enabled] AS PipelineEnabled,\r\n", "\tr.[Enabled] AS RecipientEnabled,\r\n", "\tal.[Enabled] AS AlertLinkEnabled\r\n", "FROM\r\n", "\t[procfwk].[PipelineAlertLink] al\r\n", "\tINNER JOIN [procfwk].[Recipients] r\r\n", "\t\tON al.[RecipientId] = r.[RecipientId]\r\n", "\tINNER JOIN [procfwk].[Pipelines] p\r\n", "\t\tON al.[PipelineId] = p.[PipelineId]\r\n", "\tINNER JOIN [procfwk].[AlertOutcomes] ao\r\n", "\t\tON al.[OutcomesBitValue] & ao.[BitValue] <> 0\r\n", "WHERE\r\n", "\tr.[EmailAddress] = 'test.user3@adfprocfwk.com';\r\n", "\r\n", "--Customise the email body HTML template:\r\n", "DECLARE @BodyTemplate NVARCHAR(MAX) =\r\n", "N'
Pipeline Name: ##PipelineName###
\r\n", "Status: ##Status###

\r\n", "Execution ID: ##ExecId###
\r\n", "Run ID: ##RunId###

\r\n", "Start Date Time: ##StartDateTime###
\r\n", "End Date Time: ##EndDateTime###
\r\n", "Duration (Minutes): ##Duration###

\r\n", "Called by Data Factory: ##CalledByADF###
\r\n", "Executed by Data Factory: ##ExecutedByADF###
'\r\n", "\r\n", "EXEC [procfwk].[AddProperty]\r\n", "\t@PropertyName = N'EmailAlertBodyTemplate',\r\n", "\t@PropertyValue = @BodyTemplate,\r\n", "\t@Description = N'Custom HTML template of execution information used as the eventual body in email alerts sent.';" ], "metadata": { "azdata_cell_guid": "63ffc489-41e7-4ff7-8fd8-4accc2429cf2" }, "outputs": [], "execution_count": null }, { "cell_type": "markdown", "source": [ "***\r\n", "# Resources and Content\r\n", "\r\n", "| ![alt text](https://mrpaulandrew.files.wordpress.com/2020/03/azure-square-logo.png?w=75 \"Blog Icon\") | Blogs |[mrpaulandrew.com/ADF.procfwk](https://mrpaulandrew.com/category/azure/data-factory/adf-procfwk/)|\r\n", "|:----:|:----:|:----:|\r\n", "| ![alt text](https://mrpaulandrew.files.wordpress.com/2018/11/github-icon.png?w=75 \"GitHub Icon\") | **GitHub** |**[github.com/mrpaulandrew/ADF.procfwk](https://github.com/mrpaulandrew/ADF.procfwk)** |\r\n", "| ![alt text](https://mrpaulandrew.files.wordpress.com/2020/03/twitterlogo.png?w=75 \"Twitter Icon\") | **Twitter** |**[#ADFprocfwk](https://twitter.com/search?q=%23ADFprocfwk&src=hashtag_click)** |" ], "metadata": { "azdata_cell_guid": "0ffd1628-cef6-41a4-b37c-133d37a01bcf" } } ] } ================================================ FILE: ProcessingFramework.sln ================================================  Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 16 VisualStudioVersion = 16.0.29728.190 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Functions", "Functions\Functions.csproj", "{491422B7-393B-4421-BE9E-5B70E7746CC1}" EndProject Project("{00D1A9C2-B5F0-4AF3-8072-F6C62B433612}") = "MetadataDB", "MetadataDB\MetadataDB.sqlproj", "{202EBF84-A56B-4999-92A3-10F7FFE4EF25}" EndProject Project("{151D2E53-A2C4-4D7D-83FE-D05416EBD58E}") = "DeploymentTools", "DeploymentTools\DeploymentTools.deployproj", "{3AEAD846-DBE4-45AD-97DD-37E94BE009FD}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Markdown", "Markdown", "{C7297C57-3403-473B-AA2A-31012A4A9ADF}" ProjectSection(SolutionItems) = preProject .github\ISSUE_TEMPLATE\bug-found.md = .github\ISSUE_TEMPLATE\bug-found.md CODE_OF_CONDUCT.md = CODE_OF_CONDUCT.md CONTRIBUTING.md = CONTRIBUTING.md .github\ISSUE_TEMPLATE\feature-request.md = .github\ISSUE_TEMPLATE\feature-request.md LICENSE = LICENSE README.md = README.md EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataFactory", "DataFactory", "{E081CE7B-3DE2-4AB7-8C3A-E58A075BD6F5}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "dataset", "dataset", "{997E1D61-E47F-48B0-8F67-09ED4AE92E0E}" ProjectSection(SolutionItems) = preProject DataFactory\dataset\GetSetMetadata.json = DataFactory\dataset\GetSetMetadata.json EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "linkedService", "linkedService", "{6B27406C-2C36-4BA1-AFC8-CEFC960D8EE6}" ProjectSection(SolutionItems) = preProject DataFactory\linkedService\FrameworkFunctions.json = DataFactory\linkedService\FrameworkFunctions.json DataFactory\linkedService\Keys.json = DataFactory\linkedService\Keys.json DataFactory\linkedService\SupportDatabase.json = DataFactory\linkedService\SupportDatabase.json EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "pipeline", "pipeline", "{9CEBE9B6-F1E4-43A8-B84D-FF7D43D5CB20}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "_ProcFwk", "_ProcFwk", "{17586826-8BE9-4622-B12E-1683F2A20352}" ProjectSection(SolutionItems) = preProject DataFactory\pipeline\01-Grandparent.json = DataFactory\pipeline\01-Grandparent.json DataFactory\pipeline\02-Parent.json = DataFactory\pipeline\02-Parent.json DataFactory\pipeline\03-Child.json = DataFactory\pipeline\03-Child.json DataFactory\pipeline\04-Infant.json = DataFactory\pipeline\04-Infant.json EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Workers", "Workers", "{04142829-7936-4C27-9891-FB086BEAE1AD}" ProjectSection(SolutionItems) = preProject DataFactory\pipeline\Intentional Error.json = DataFactory\pipeline\Intentional Error.json DataFactory\pipeline\Wait 1.json = DataFactory\pipeline\Wait 1.json DataFactory\pipeline\Wait 10.json = DataFactory\pipeline\Wait 10.json DataFactory\pipeline\Wait 2.json = DataFactory\pipeline\Wait 2.json DataFactory\pipeline\Wait 3.json = DataFactory\pipeline\Wait 3.json DataFactory\pipeline\Wait 4.json = DataFactory\pipeline\Wait 4.json DataFactory\pipeline\Wait 5.json = DataFactory\pipeline\Wait 5.json DataFactory\pipeline\Wait 6.json = DataFactory\pipeline\Wait 6.json DataFactory\pipeline\Wait 7.json = DataFactory\pipeline\Wait 7.json DataFactory\pipeline\Wait 8.json = DataFactory\pipeline\Wait 8.json DataFactory\pipeline\Wait 9.json = DataFactory\pipeline\Wait 9.json EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Images", "Images", "{16E32CF7-361C-4A58-B0E1-5C8C46181ADE}" ProjectSection(SolutionItems) = preProject Images\Activity Chain.png = Images\Activity Chain.png Images\Database Diagram.png = Images\Database Diagram.png Images\procfwk Designs.vsdx = Images\procfwk Designs.vsdx Images\Repo Social Media Image.png = Images\Repo Social Media Image.png EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Notebooks", "Notebooks", "{19F99FD2-9170-42DA-B4FE-A52A358B5473}" ProjectSection(SolutionItems) = preProject Notebooks\Databricks - Throw Exception.scala = Notebooks\Databricks - Throw Exception.scala Notebooks\Metadata Guide and Handy Code Snippets.ipynb = Notebooks\Metadata Guide and Handy Code Snippets.ipynb EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ARM Templates", "ARM Templates", "{CC7B607B-8C84-4E07-A6C8-326923752F92}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataFactory", "DataFactory", "{C8F387D0-13CC-4AA5-892C-91D2F3403B55}" ProjectSection(SolutionItems) = preProject ARM Templates\Data Factory\v1.0 Export.json = ARM Templates\Data Factory\v1.0 Export.json ARM Templates\Data Factory\v1.1 Export.json = ARM Templates\Data Factory\v1.1 Export.json ARM Templates\Data Factory\v1.2 Export.json = ARM Templates\Data Factory\v1.2 Export.json ARM Templates\Data Factory\v1.3 Export.json = ARM Templates\Data Factory\v1.3 Export.json ARM Templates\Data Factory\v1.4 Export.json = ARM Templates\Data Factory\v1.4 Export.json ARM Templates\Data Factory\v1.5 Export.json = ARM Templates\Data Factory\v1.5 Export.json ARM Templates\Data Factory\v1.6 Export.json = ARM Templates\Data Factory\v1.6 Export.json ARM Templates\Data Factory\v1.7 Export.json = ARM Templates\Data Factory\v1.7 Export.json ARM Templates\Data Factory\v1.8 Export.json = ARM Templates\Data Factory\v1.8 Export.json ARM Templates\Data Factory\v1.8.3 Export.json = ARM Templates\Data Factory\v1.8.3 Export.json ARM Templates\Data Factory\v1.8.5 Export.json = ARM Templates\Data Factory\v1.8.5 Export.json ARM Templates\Data Factory\v1.8.6 Export.json = ARM Templates\Data Factory\v1.8.6 Export.json ARM Templates\Data Factory\v1.9 Export.json = ARM Templates\Data Factory\v1.9 Export.json ARM Templates\Data Factory\v1.9.1 Export.json = ARM Templates\Data Factory\v1.9.1 Export.json ARM Templates\Data Factory\v1.9.2 Export.json = ARM Templates\Data Factory\v1.9.2 Export.json ARM Templates\Data Factory\v2.0 Export.json = ARM Templates\Data Factory\v2.0 Export.json EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "trigger", "trigger", "{FC7877EF-110F-4AD7-B203-EEECE6F08A86}" ProjectSection(SolutionItems) = preProject DataFactory\trigger\FunctionalTestingTrigger.json = DataFactory\trigger\FunctionalTestingTrigger.json EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Reporting", "Reporting", "{5A56A63A-351D-467D-A2DA-FDB3C9CBD252}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "PowerBI", "PowerBI", "{EBC230DF-D45D-43E4-A74E-32ADA3DDAE7A}" ProjectSection(SolutionItems) = preProject Reporting\PowerBI\Executions Overview.pbit = Reporting\PowerBI\Executions Overview.pbit Reporting\PowerBI\Executions Overview.pbix = Reporting\PowerBI\Executions Overview.pbix EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SQL Database", "SQL Database", "{20313ECA-118E-408D-BB42-8094A7A9CD20}" ProjectSection(SolutionItems) = preProject ARM Templates\SQL Database\v1.6 Export.json = ARM Templates\SQL Database\v1.6 Export.json EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Functions App", "Functions App", "{463330CD-189F-42B4-984A-8D3C1385645B}" ProjectSection(SolutionItems) = preProject ARM Templates\Functions App\v1.6 Export.json = ARM Templates\Functions App\v1.6 Export.json EndProjectSection EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FactoryTesting", "FactoryTesting\FactoryTesting.csproj", "{777BB424-9D89-49E5-BE29-E73E0A0E159A}" EndProject Project("{00D1A9C2-B5F0-4AF3-8072-F6C62B433612}") = "MetadataDBTests", "MetadataDBTests\MetadataDBTests.sqlproj", "{32D54B90-932D-44A3-911D-007C4B90BC55}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "_ProcFwkUtils", "_ProcFwkUtils", "{6809EDBD-125D-440E-8486-9CFAB9BB5B7D}" ProjectSection(SolutionItems) = preProject DataFactory\pipeline\Check For Running Pipeline.json = DataFactory\pipeline\Check For Running Pipeline.json DataFactory\pipeline\Email Sender.json = DataFactory\pipeline\Email Sender.json DataFactory\pipeline\Intentional Error.json = DataFactory\pipeline\Intentional Error.json DataFactory\pipeline\Throw Exception.json = DataFactory\pipeline\Throw Exception.json EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Synapse", "Synapse", "{6B8CC8C2-C4AA-4BC4-85EC-719348FC3B07}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "pipeline", "pipeline", "{398CB5BE-3687-4F95-8EAA-54EE0DAC4E6C}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "_ProcFwk", "_ProcFwk", "{2181D7DA-8237-418F-8CD9-CD1BF9843524}" ProjectSection(SolutionItems) = preProject Synapse\pipeline\01-Grandparent.json = Synapse\pipeline\01-Grandparent.json Synapse\pipeline\02-Parent.json = Synapse\pipeline\02-Parent.json Synapse\pipeline\03-Child.json = Synapse\pipeline\03-Child.json Synapse\pipeline\04-Infant.json = Synapse\pipeline\04-Infant.json EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "_ProcFwkUtils", "_ProcFwkUtils", "{6E660431-66E0-4B1A-BCFC-8FCBF1C646F2}" ProjectSection(SolutionItems) = preProject Synapse\pipeline\Check For Running Pipeline.json = Synapse\pipeline\Check For Running Pipeline.json Synapse\pipeline\Email Sender.json = Synapse\pipeline\Email Sender.json Synapse\pipeline\Throw Exception.json = Synapse\pipeline\Throw Exception.json EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Workers", "Workers", "{21058901-FBEE-4D40-83E0-21B05636417B}" ProjectSection(SolutionItems) = preProject Synapse\pipeline\Intentional Error.json = Synapse\pipeline\Intentional Error.json Synapse\pipeline\Wait 1.json = Synapse\pipeline\Wait 1.json Synapse\pipeline\Wait 10.json = Synapse\pipeline\Wait 10.json Synapse\pipeline\Wait 2.json = Synapse\pipeline\Wait 2.json Synapse\pipeline\Wait 3.json = Synapse\pipeline\Wait 3.json Synapse\pipeline\Wait 4.json = Synapse\pipeline\Wait 4.json Synapse\pipeline\Wait 5.json = Synapse\pipeline\Wait 5.json Synapse\pipeline\Wait 6.json = Synapse\pipeline\Wait 6.json Synapse\pipeline\Wait 7.json = Synapse\pipeline\Wait 7.json Synapse\pipeline\Wait 8.json = Synapse\pipeline\Wait 8.json Synapse\pipeline\Wait 9.json = Synapse\pipeline\Wait 9.json EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "dataset", "dataset", "{7F17593A-2CEF-4741-A4B7-C1CC79F12EB7}" ProjectSection(SolutionItems) = preProject Synapse\dataset\GetSetMetadata.json = Synapse\dataset\GetSetMetadata.json EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "linkedservice", "linkedservice", "{F495E5FB-B908-4F76-B8BB-6C83CC5D2337}" ProjectSection(SolutionItems) = preProject Synapse\linkedService\FrameworkFunctions.json = Synapse\linkedService\FrameworkFunctions.json Synapse\linkedService\Keys.json = Synapse\linkedService\Keys.json Synapse\linkedService\SupportDatabase.json = Synapse\linkedService\SupportDatabase.json EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "trigger", "trigger", "{29DBE5FC-D1FE-4C49-9DA2-4115C7F830B4}" ProjectSection(SolutionItems) = preProject Synapse\trigger\FunctionalTestingTrigger.json = Synapse\trigger\FunctionalTestingTrigger.json EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Synapse", "Synapse", "{4A3176F1-B8E0-4D2B-861B-9A1655EB322E}" ProjectSection(SolutionItems) = preProject ARM Templates\Synapse\v2.0 Export.json = ARM Templates\Synapse\v2.0 Export.json EndProjectSection EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU Release|Any CPU = Release|Any CPU EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {491422B7-393B-4421-BE9E-5B70E7746CC1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {491422B7-393B-4421-BE9E-5B70E7746CC1}.Debug|Any CPU.Build.0 = Debug|Any CPU {491422B7-393B-4421-BE9E-5B70E7746CC1}.Release|Any CPU.ActiveCfg = Release|Any CPU {491422B7-393B-4421-BE9E-5B70E7746CC1}.Release|Any CPU.Build.0 = Release|Any CPU {202EBF84-A56B-4999-92A3-10F7FFE4EF25}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {202EBF84-A56B-4999-92A3-10F7FFE4EF25}.Debug|Any CPU.Build.0 = Debug|Any CPU {202EBF84-A56B-4999-92A3-10F7FFE4EF25}.Debug|Any CPU.Deploy.0 = Debug|Any CPU {202EBF84-A56B-4999-92A3-10F7FFE4EF25}.Release|Any CPU.ActiveCfg = Release|Any CPU {202EBF84-A56B-4999-92A3-10F7FFE4EF25}.Release|Any CPU.Build.0 = Release|Any CPU {202EBF84-A56B-4999-92A3-10F7FFE4EF25}.Release|Any CPU.Deploy.0 = Release|Any CPU {3AEAD846-DBE4-45AD-97DD-37E94BE009FD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {3AEAD846-DBE4-45AD-97DD-37E94BE009FD}.Debug|Any CPU.Build.0 = Debug|Any CPU {3AEAD846-DBE4-45AD-97DD-37E94BE009FD}.Release|Any CPU.ActiveCfg = Release|Any CPU {3AEAD846-DBE4-45AD-97DD-37E94BE009FD}.Release|Any CPU.Build.0 = Release|Any CPU {777BB424-9D89-49E5-BE29-E73E0A0E159A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {777BB424-9D89-49E5-BE29-E73E0A0E159A}.Debug|Any CPU.Build.0 = Debug|Any CPU {777BB424-9D89-49E5-BE29-E73E0A0E159A}.Release|Any CPU.ActiveCfg = Release|Any CPU {777BB424-9D89-49E5-BE29-E73E0A0E159A}.Release|Any CPU.Build.0 = Release|Any CPU {32D54B90-932D-44A3-911D-007C4B90BC55}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {32D54B90-932D-44A3-911D-007C4B90BC55}.Debug|Any CPU.Build.0 = Debug|Any CPU {32D54B90-932D-44A3-911D-007C4B90BC55}.Debug|Any CPU.Deploy.0 = Debug|Any CPU {32D54B90-932D-44A3-911D-007C4B90BC55}.Release|Any CPU.ActiveCfg = Release|Any CPU {32D54B90-932D-44A3-911D-007C4B90BC55}.Release|Any CPU.Build.0 = Release|Any CPU {32D54B90-932D-44A3-911D-007C4B90BC55}.Release|Any CPU.Deploy.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection GlobalSection(NestedProjects) = preSolution {997E1D61-E47F-48B0-8F67-09ED4AE92E0E} = {E081CE7B-3DE2-4AB7-8C3A-E58A075BD6F5} {6B27406C-2C36-4BA1-AFC8-CEFC960D8EE6} = {E081CE7B-3DE2-4AB7-8C3A-E58A075BD6F5} {9CEBE9B6-F1E4-43A8-B84D-FF7D43D5CB20} = {E081CE7B-3DE2-4AB7-8C3A-E58A075BD6F5} {17586826-8BE9-4622-B12E-1683F2A20352} = {9CEBE9B6-F1E4-43A8-B84D-FF7D43D5CB20} {04142829-7936-4C27-9891-FB086BEAE1AD} = {9CEBE9B6-F1E4-43A8-B84D-FF7D43D5CB20} {C8F387D0-13CC-4AA5-892C-91D2F3403B55} = {CC7B607B-8C84-4E07-A6C8-326923752F92} {FC7877EF-110F-4AD7-B203-EEECE6F08A86} = {E081CE7B-3DE2-4AB7-8C3A-E58A075BD6F5} {EBC230DF-D45D-43E4-A74E-32ADA3DDAE7A} = {5A56A63A-351D-467D-A2DA-FDB3C9CBD252} {20313ECA-118E-408D-BB42-8094A7A9CD20} = {CC7B607B-8C84-4E07-A6C8-326923752F92} {463330CD-189F-42B4-984A-8D3C1385645B} = {CC7B607B-8C84-4E07-A6C8-326923752F92} {6809EDBD-125D-440E-8486-9CFAB9BB5B7D} = {17586826-8BE9-4622-B12E-1683F2A20352} {398CB5BE-3687-4F95-8EAA-54EE0DAC4E6C} = {6B8CC8C2-C4AA-4BC4-85EC-719348FC3B07} {2181D7DA-8237-418F-8CD9-CD1BF9843524} = {398CB5BE-3687-4F95-8EAA-54EE0DAC4E6C} {6E660431-66E0-4B1A-BCFC-8FCBF1C646F2} = {2181D7DA-8237-418F-8CD9-CD1BF9843524} {21058901-FBEE-4D40-83E0-21B05636417B} = {398CB5BE-3687-4F95-8EAA-54EE0DAC4E6C} {7F17593A-2CEF-4741-A4B7-C1CC79F12EB7} = {6B8CC8C2-C4AA-4BC4-85EC-719348FC3B07} {F495E5FB-B908-4F76-B8BB-6C83CC5D2337} = {6B8CC8C2-C4AA-4BC4-85EC-719348FC3B07} {29DBE5FC-D1FE-4C49-9DA2-4115C7F830B4} = {6B8CC8C2-C4AA-4BC4-85EC-719348FC3B07} {4A3176F1-B8E0-4D2B-861B-9A1655EB322E} = {CC7B607B-8C84-4E07-A6C8-326923752F92} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {170F5302-BB9C-4569-8F43-D859C7678EF0} EndGlobalSection EndGlobal ================================================ FILE: README.md ================================================ # Read Me - Orchestrate.[procfwk](http://procfwk.com/) For complete documentation on this solution see [procfwk.com](http://procfwk.com/). ## ProcFwk Has Become CF.Cumulus.Control See blog: [mrpaulandrew.com](https://mrpaulandrew.com/2024/01/07/procfwk-is-getting-an-upgrade-to-cf-cumulus/) See new product page: [cloudformations.org/cumulus](https://www.cloudformations.org/cumulus?utm_source=pa&utm_medium=github&utm_campaign=cumulus&utm_content=l2) [ ![](https://mrpaulandrew.github.io/procfwk/procfwk-to-cumulus.png) ](https://mrpaulandrew.github.io/procfwk/procfwk-to-cumulus.png) ProcFwk will receive no further development beyond December 2023. ## Framework Capabilities * Granular metadata control. * Metadata integrity checking. * Global properties. * Complete pipeline dependency chains. * Concurrent batch executions (hourly/daily/monthly). * Execution restart-ability. * Parallel pipeline execution. * Full execution and error logs. * Operational dashboards. * Low cost orchestration. * Disconnection between framework and Worker pipelines. * Cross Tenant/Subscription/Data Factory control flows. * Pipeline parameter support. * Simple troubleshooting. * Easy deployment. * Email alerting. * Automated testing. * Azure Key Vault integration. * Is pipeline already running checks. ## Complete Data Factory Activity Chain [ ![](https://mrpaulandrew.github.io/procfwk/activitychain-full.png) ](https://mrpaulandrew.github.io/procfwk/activitychain-full.png) ## Issues If you've found a bug or have a new feature request please log the details using the repository issues. Go to... [Issues](https://github.com/mrpaulandrew/procfwk/issues) ## Projects Go to... [External Requests](https://github.com/mrpaulandrew/procfwk/projects/2) Go to... [Internal Backlog](https://github.com/mrpaulandrew/procfwk/projects/1) ## Release Details | Version | Overview | Version Details & Release Notes | |:----:|--------------|--------| | 2.0 |Azure Synapse Analytics fully supported as an interchangeable orchestrator of pipelines within the procfwk.|GitHub Pages:
[Orchestrators](https://mrpaulandrew.github.io/procfwk/orchestrators)
[Orchestrator Types](https://mrpaulandrew.github.io/procfwk/orchestratortypes)

Release Summary Video:
[YouTube - procfwk Playlist](https://www.youtube.com/c/mrpaulandrew)

GitHub Issues:
[procfwk #95](https://github.com/mrpaulandrew/procfwk/issues/95) | | 2.0-beta |Azure Synapse Analytics **Beta** support added.

Development of Azure Functions App completed using the Synapse namespace: _Azure.Analytics.Synapse.Artifacts_ with version **1.0.0-beta.1** of the NuGet package.|GitHub Issues:
[procfwk #21](https://github.com/mrpaulandrew/procfwk/issues/21) | | 1.9.2 |Batch Executions added, plus:
  • Exception Pipeline
  • Running Pipeline Check
  • Pipeline Parameter Last Values
  • Worker Pipeline Validation
|GitHub Pages: [Batch Executions](https://mrpaulandrew.github.io/procfwk/executionbatches)

Release Demo Summary Video: [YouTube - procfwk Playlist](https://www.youtube.com/c/mrpaulandrew)

GitHub Issues:
[procfwk #78](https://github.com/mrpaulandrew/procfwk/issues/78)
[procfwk #77](https://github.com/mrpaulandrew/procfwk/issues/77)
[procfwk #71](https://github.com/mrpaulandrew/procfwk/issues/71)
[procfwk #73](https://github.com/mrpaulandrew/procfwk/issues/73)
[procfwk #80](https://github.com/mrpaulandrew/procfwk/issues/80)
[procfwk #72](https://github.com/mrpaulandrew/procfwk/issues/72) | | 1.9.1 |Activity Policy Update, plus:
  • Secure Activity Inputs/Outputs.
  • Execution Wrapper Hardening.
  • New Activity Icons and Framework Factory Cosmetics.
|GitHub Issues:
[procfwk #65](https://github.com/mrpaulandrew/procfwk/issues/65)
[procfwk #66](https://github.com/mrpaulandrew/procfwk/issues/66)
[procfwk #67](https://github.com/mrpaulandrew/procfwk/issues/67)
[procfwk #69](https://github.com/mrpaulandrew/procfwk/issues/69) | | 1.9.0 |Cross Tenant & Subscription Support added, plus:
  • New integration tests created.
  • Infant pipeline refactoring.
  • tSQLt project added.
|GitHub Issues:
[procfwk #34](https://github.com/mrpaulandrew/procfwk/issues/34)
[procfwk #35](https://github.com/mrpaulandrew/procfwk/issues/35)
[procfwk #46](https://github.com/mrpaulandrew/procfwk/issues/46)
[procfwk #55](https://github.com/mrpaulandrew/procfwk/issues/55)
[procfwk #56](https://github.com/mrpaulandrew/procfwk/issues/56)
[procfwk #59](https://github.com/mrpaulandrew/procfwk/issues/59) | | 1.8.6 |Pipeline Expressions Refactored to Use Variables added, plus:
  • New integration tests created.
  • Complete activity chain redrawn in Visio.
|GitHub Issues:
[procfwk #51](https://github.com/mrpaulandrew/procfwk/issues/51)
[procfwk #52](https://github.com/mrpaulandrew/procfwk/issues/52) | | 1.8.5 |Execution Precursor added, plus:
  • PowerShell helper to add initial Worker metadata.
|[procfwk v1.8.5 - Execution Precursor](https://mrpaulandrew.com/2020/08/17/adf-procfwk-v1-8-5-execution-precursor/) | | 1.8.4 |Database Schema Reorganise and Restructuring |[procfwk v1.8.4 - Database Schema Reorganise and Restructuring](https://mrpaulandrew.com/2020/07/23/adf-procfwk-v1-8-4-database-schema-reorganise-and-restructuring/) | | 1.8.3 |Bug Fixes from the Community, including:
  • Email alerts sent to blank email addresses due to wrong flow in Child pipeline.
  • Worker pipelines cancelled during an execution fail when the framework is restarted due to missing Parent pipeline clean up condition.
|GitHub Issues:
[procfwk #38](https://github.com/mrpaulandrew/procfwk/issues/38)
[procfwk #37](https://github.com/mrpaulandrew/procfwk/issues/37) | | 1.8.2 |Optionally Store SPN Details in Azure Key Vault |[procfwk v1.8.2 - Optionally Store SPN Details in Azure Key Vault](https://mrpaulandrew.com/2020/07/22/adf-procfwk-v1-8-2-optionally-store-spn-details-in-azure-key-vault/) | | 1.8.1 |Automated Framework Pipeline Testing added, including tests for:
  • A simple grandparent run.
  • All types of failure dependency handling.
  • Metadata checks when pipelines and staged are disabled.
  • No pipeline parameters provided.
|Blog Series:
  1. [Set up automated testing for Azure Data Factory](https://richardswinbank.net/adf/set_up_automated_testing_for_azure_data_factory)
  2. [Automate integration tests in Azure Data Factory](https://richardswinbank.net/adf/automate_integration_tests_in_azure_data_factory)
  3. [Isolated functional tests for Azure Data Factory](https://richardswinbank.net/adf/isolated_functional_tests_for_azure_data_factory)
  4. [Testing Azure Data Factory in your CI/CD pipeline](https://richardswinbank.net/adf/testing_azure_data_factory_in_your_cicd_pipeline)
  5. [Unit testing Azure Data Factory pipelines](https://richardswinbank.net/adf/unit_testing_azure_data_factory_pipelines)
  6. [Calculating Azure Data Factory test coverage](https://richardswinbank.net/adf/calculating_azure_data_factory_test_coverage)
| | 1.8.0 |Complete Pipeline Dependency Chains For Failure Handling added, plus:
  • Clean up of a previous execution run if Workers appear as running.
  • New metadata integrity checks.
  • Internal get property value function added.
|[procfwk v1.8 - Complete Pipeline Dependency Chains For Failure Handling](https://mrpaulandrew.com/2020/07/01/adf-procfwk-v1-8-complete-pipeline-dependency-chains-for-failure-handling/) | | 1.7.3 |Data Factory Deployment Updated To Use azure.datafactory.tools PowerShell Module |[SQLPlayer/azure.datafactory.tools](https://github.com/SQLPlayer/azure.datafactory.tools) | | 1.7.2 |Pipeline Parameter NULL Handling added, plus:
  • Worker pipelines with a status of 'Running' protected from a new execution start/restart.
|[procfwk v1.7.2 - NULL Pipeline Parameters Handled](https://mrpaulandrew.com/2020/06/22/adf-procfwk-v1-7-2-null-pipeline-parameters-handled/) | | 1.7.1 |Alerting Check Bug Fix added, plus:
  • Pipeline parameter value size limit removed.
|[procfwk v1.7.1 - Alerting Bug Fix And Pipeline Parameter Size Limit Removed](https://mrpaulandrew.com/2020/06/12/adf-procfwk-v1-7-1-alerting-bug-fix-and-pipeline-parameter-size-limit-removed/) | | 1.7.0 |Pipleline EMail Alerting added, plus:
  • Send email Function implemented and hardened.
  • Handy Notebook updates.
  • Activity failure paths improved.
  • MIT license and code of conduct added.
  • Error table bug fix. Error code attribute; INT to VARCHAR
|[procfwk v1.7 - Pipeline Email Alerting](https://mrpaulandrew.com/2020/06/08/adf-procfwk-v1-7-pipeline-email-alerting/) | | 1.6.0 |Error Details for Failed Activities Captured, plus:
  • Pipeline parameters used at runtime captured in execution logs.
  • Emailing Function added, not yet implemented.
  • Unknown Worker outcomes optionally blocks downstream stages.
  • Solution housekeeping.
|[procfwk v1.6 - Error Details for Failed Activities Captured](https://mrpaulandrew.com/2020/05/19/adf-procfwk-v1-6-error-details-for-failed-activities-captured/) | | 1.5.0 |Power BI Dashboard for Framework Executions, plus:
  • Worker Parallelism View.
  • Pipeline Run ID now logged.
  • Logging Attributes Bug Fix.
|[procfwk v1.5 - Power BI Dashboard for Framework Executions](https://mrpaulandrew.com/2020/05/01/adf-procfwk-v1-5-power-bi-dashboard-for-framework-executions/) | | 1.4.0 |Enhancements for Long Running Pipelines, plus:
  • Pipeline check status function added.
  • Function Data Factory client moved to internal class.
  • SQL GETDATE() changed to GETUTCDATE().
  • Glossary created, [here](https://github.com/mrpaulandrew/procfwk/blob/master/Glossary.md).
  • Updated database views.
|[procfwk v1.4 - Enhancements for Long Running Pipelines](https://mrpaulandrew.com/2020/04/20/adf-procfwk-v1-4-enhancements-for-long-running-pipelines/) | | 1.3.0 |Metadata Integrity Checks, plus:
  • Logical pipeline predecessors.
  • Data Factory Powershell deployment script.
  • Helper Notebook.
  • Database objects renames and solution tidy up.
|[procfwk v1.3 - Metadata Integrity Checks](https://mrpaulandrew.com/2020/04/07/adf-procfwk-v1-3-metadata-integrity-checks/) | | 1.2.0 |Execution Restartability, plus:
  • Data Factory annotations and descriptions.
  • Database covering indexes.
  • Pipeline log status changed from 'Started' to 'Preparing'.
  • Pipeline log start date/time now set in child pipeline.
|[procfwk v1.2 - Execution Restartability](https://mrpaulandrew.com/2020/03/24/adf-procfwk-v1-2-execution-restartability/) | | 1.1.0 |Service Principal Handling via Metadata, plus:
  • Data Factory table.
  • Properties table and view.
  • Function body bug fix.
  • New sample data.
|[procfwk v1.1 - Service Principal Handling via Metadata](https://mrpaulandrew.com/2020/03/17/adf-procfwk-v1-1-service-principal-handling-via-metadata/) | | 1.0.0 |Simple framework designed and base compontents built.
  • Part 1 - Design, concepts, service coupling, caveats, problems.
  • Part 2 - Database build and metadata.
  • Part 3 - Data Factory build.
  • Part 4 - Execution, conclusions, enhancements.
|Blog Series:
[Creating a Simple Staged Metadata Driven Processing Framework for Azure Data Factory Pipelines](https://mrpaulandrew.com/2020/02/25/creating-a-simple-staged-metadata-driven-processing-framework-for-azure-data-factory-pipelines-part-1-of-4/) | ================================================ FILE: Synapse/dataflow/MDF _Order_Count.json ================================================ { "name": "MDF _Order_Count", "properties": { "type": "MappingDataFlow", "typeProperties": { "sources": [], "sinks": [], "transformations": [], "script": "" } } } ================================================ FILE: Synapse/dataset/GetSetMetadata.json ================================================ { "name": "GetSetMetadata", "properties": { "description": "Single generic dataset used to get and set all database metadata.", "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" }, "folder": { "name": "_ProcFwk" }, "annotations": [ "procfwk" ], "type": "AzureSqlTable", "schema": [] } } ================================================ FILE: Synapse/factory/FrameworkFactory.json ================================================ { "name": "FrameworkFactory" } ================================================ FILE: Synapse/integrationRuntime/AutoResolveIntegrationRuntime.json ================================================ { "name": "AutoResolveIntegrationRuntime", "properties": { "type": "Managed", "typeProperties": { "computeProperties": { "location": "AutoResolve", "dataFlowProperties": { "computeType": "General", "coreCount": 8, "timeToLive": 0 } } } } } ================================================ FILE: Synapse/linkedService/FrameworkFunctions.json ================================================ { "name": "FrameworkFunctions", "properties": { "annotations": [ "procfwk" ], "type": "AzureFunction", "typeProperties": { "functionAppUrl": "https://frameworksupportfunctions.azurewebsites.net", "functionKey": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "FrameworkFunctionsKey" } }, "description": "Interact with the Azure Functions App used as middle ware when making requests to Worker pipelines. Authentication done at the Function App level." } } ================================================ FILE: Synapse/linkedService/Keys.json ================================================ { "name": "Keys", "properties": { "annotations": [ "procfwk" ], "type": "AzureKeyVault", "typeProperties": { "baseUrl": "https://FrameworkKeys.vault.azure.net/" }, "description": "Connection to Key Vault for all other ADF linked service credentials required to run the processing framework." } } ================================================ FILE: Synapse/linkedService/SupportDatabase.json ================================================ { "name": "SupportDatabase", "properties": { "description": "Connection between ADF and processing framework metadata SQLDB.", "annotations": [ "procfwk" ], "type": "AzureSqlDatabase", "typeProperties": { "connectionString": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "Keys", "type": "LinkedServiceReference" }, "secretName": "FrameworkMetadataDev" } } } } ================================================ FILE: Synapse/linkedService/covid-tracking.json ================================================ { "name": "covid-tracking", "properties": { "annotations": [], "type": "AzureBlobStorage", "typeProperties": { "sasUri": "https://pandemicdatalake.blob.core.windows.net/public/curated/covid-19/covid_tracking/latest/covid_tracking.parquet?\"\"" } } } ================================================ FILE: Synapse/linkedService/procfwkforsynapse-WorkspaceDefaultSqlServer.json ================================================ { "name": "procfwkforsynapse-WorkspaceDefaultSqlServer", "type": "Microsoft.Synapse/workspaces/linkedservices", "properties": { "typeProperties": { "connectionString": "Data Source=tcp:procfwkforsynapse.sql.azuresynapse.net,1433;Initial Catalog=@{linkedService().DBName}" }, "parameters": { "DBName": { "type": "String" } }, "type": "AzureSqlDW", "connectVia": { "referenceName": "AutoResolveIntegrationRuntime", "type": "IntegrationRuntimeReference" }, "annotations": [] } } ================================================ FILE: Synapse/linkedService/procfwkforsynapse-WorkspaceDefaultStorage.json ================================================ { "name": "procfwkforsynapse-WorkspaceDefaultStorage", "type": "Microsoft.Synapse/workspaces/linkedservices", "properties": { "typeProperties": { "url": "https://frameworkdatalake01.dfs.core.windows.net" }, "type": "AzureBlobFS", "connectVia": { "referenceName": "AutoResolveIntegrationRuntime", "type": "IntegrationRuntimeReference" }, "annotations": [] } } ================================================ FILE: Synapse/notebook/Getting Started with Delta Lake.json ================================================ { "name": "Getting Started with Delta Lake", "properties": { "nbformat": 4, "nbformat_minor": 2, "bigDataPool": { "referenceName": "SampleSpark", "type": "BigDataPoolReference" }, "sessionProperties": { "driverMemory": "28g", "driverCores": 4, "executorMemory": "28g", "executorCores": 4, "numExecutors": 2, "conf": { "spark.dynamicAllocation.enabled": "false", "spark.dynamicAllocation.minExecutors": "2", "spark.dynamicAllocation.maxExecutors": "2" } }, "metadata": { "saveOutput": true, "kernelspec": { "name": "synapse_sparkdotnet", "display_name": "Synapse SparkDotNet" }, "language_info": { "name": "csharp" }, "a365ComputeOptions": { "id": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/ADF.procfwk/providers/Microsoft.Synapse/workspaces/procfwkforsynapse/bigDataPools/SampleSpark", "name": "SampleSpark", "type": "Spark", "endpoint": "https://procfwkforsynapse.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/SampleSpark", "auth": { "type": "AAD", "authResource": "https://dev.azuresynapse.net" }, "sparkVersion": "2.4", "nodeCount": 3, "cores": 8, "memory": 56, "automaticScaleJobs": false }, "sessionKeepAliveTimeout": 30 }, "cells": [ { "cell_type": "markdown", "source": [ "# Hitchhiker's Guide to Delta Lake (.NET for Spark C#)\n", "\n", "This tutorial has been adapted for more clarity from its original counterpart [here](https://docs.delta.io/latest/quick-start.html). This notebook helps you quickly explore the main features of [Delta Lake](https://github.com/delta-io/delta). It provides code snippets that show how to read from and write to Delta Lake tables from interactive, batch, and streaming queries.\n", "\n", "Here's what we will cover:\n", "* Create a table\n", "* Understanding meta-data\n", "* Read data\n", "* Update table data\n", "* Overwrite table data\n", "* Conditional update without overwrite\n", "* Read older versions of data using Time Travel\n", "* Write a stream of data to a table\n", "* Read a stream of changes from a table" ] }, { "cell_type": "markdown", "source": [ "## Configuration\n", "Make sure you modify this as appropriate." ] }, { "cell_type": "code", "source": [ "var sessionId = (new Random()).Next(10000000);\n", "var deltaTablePath = $\"/delta/delta-table-{sessionId}\";\n", "\n", "deltaTablePath" ], "execution_count": 1 }, { "cell_type": "markdown", "source": [ "## Create a table\n", "To create a Delta Lake table, write a DataFrame out in the **delta** format. You can use existing Spark SQL code and change the format from parquet, csv, json, and so on, to delta.\n", "\n", "These operations create a new Delta Lake table using the schema that was inferred from your DataFrame. For the full set of options available when you create a new Delta Lake table, see Create a table and Write to a table (subsequent cells in this notebook)." ] }, { "cell_type": "code", "source": [ "var data = spark.Range(0,5);\n", "data.Show();\n", "data.Write().Format(\"delta\").Save(deltaTablePath);" ], "execution_count": 2 }, { "cell_type": "markdown", "source": [ "## Understanding Meta-data\n", "\n", "In Delta Lake, meta-data is no different from data i.e., it is stored next to the data. Therefore, an interesting side-effect here is that you can peek into meta-data using regular Spark APIs. " ] }, { "cell_type": "code", "source": [ "using System.Linq;\n", "spark.Read().Text($\"{deltaTablePath}/_delta_log/\").Collect().ToList().ForEach(x => Console.WriteLine(x.GetAs(\"value\")));" ], "execution_count": 3 }, { "cell_type": "markdown", "source": [ "## Read data\n", "\n", "You read data in your Delta Lake table by specifying the path to the files." ] }, { "cell_type": "code", "source": [ "var df = spark.Read().Format(\"delta\").Load(deltaTablePath);\n", "df.Show()" ], "execution_count": 4 }, { "cell_type": "markdown", "source": [ "## Update table data\n", "\n", "Delta Lake supports several operations to modify tables using standard DataFrame APIs. This example runs a batch job to overwrite the data in the table.\n", "" ] }, { "cell_type": "code", "source": [ "var data = spark.Range(5,10);\n", "data.Write().Format(\"delta\").Mode(\"overwrite\").Save(deltaTablePath);\n", "df.Show();" ], "execution_count": 5 }, { "cell_type": "markdown", "source": [ "When you now inspect the meta-data, what you will notice is that the original data is over-written. Well, not in a true sense but appropriate entries are added to Delta's transaction log so it can provide an \"illusion\" that the original data was deleted. We can verify this by re-inspecting the meta-data. You will see several entries indicating reference removal to the original data." ] }, { "cell_type": "code", "source": [ "spark.Read().Text($\"{deltaTablePath}/_delta_log/\").Collect().ToList().ForEach(x => Console.WriteLine(x.GetAs(\"value\")));" ], "execution_count": 6 }, { "cell_type": "markdown", "source": [ "## Save as catalog tables\n", "\n", "Delta Lake can write to managed or external catalog tables.\n", "" ] }, { "cell_type": "code", "source": [ "// Write data to a new managed catalog table.\n", "data.Write().Format(\"delta\").SaveAsTable(\"ManagedDeltaTable\");" ], "execution_count": 7 }, { "cell_type": "code", "source": [ "// Define an external catalog table that points to the existing Delta Lake data in storage.\n", "spark.Sql($\"CREATE TABLE ExternalDeltaTable USING DELTA LOCATION '{deltaTablePath}'\");" ], "execution_count": 8 }, { "cell_type": "code", "source": [ "// List the 2 new tables.\n", "spark.Sql(\"SHOW TABLES\").Show();\n", "\n", "// Explore their properties.\n", "spark.Sql(\"DESCRIBE EXTENDED ManagedDeltaTable\").Show(truncate: 0);\n", "spark.Sql(\"DESCRIBE EXTENDED ExternalDeltaTable\").Show(truncate: 0);" ], "execution_count": 9 }, { "cell_type": "markdown", "source": [ "## Conditional update without overwrite\n", "\n", "Delta Lake provides programmatic APIs to conditional update, delete, and merge (upsert) data into tables. For more information on these operations, see [Table Deletes, Updates, and Merges](https://docs.delta.io/latest/delta-update.html)." ] }, { "cell_type": "code", "source": [ "using Microsoft.Spark.Extensions.Delta;\n", "using Microsoft.Spark.Extensions.Delta.Tables;\n", "using Microsoft.Spark.Sql;\n", "using static Microsoft.Spark.Sql.Functions;\n", "\n", "var deltaTable = DeltaTable.ForPath(deltaTablePath);" ], "execution_count": 10 }, { "cell_type": "code", "source": [ "// Update every even value by adding 100 to it\n", "deltaTable.Update(\n", " condition: Expr(\"id % 2 == 0\"),\n", " set: new Dictionary(){{ \"id\", Expr(\"id + 100\") }});\n", "deltaTable.ToDF().Show();" ], "execution_count": 11 }, { "cell_type": "code", "source": [ "// Delete every even value\n", "deltaTable.Delete(condition: Expr(\"id % 2 == 0\"));\n", "deltaTable.ToDF().Show();" ], "execution_count": 12 }, { "cell_type": "code", "source": [ "// Upsert (merge) new data\n", "var newData = spark.Range(20).As(\"newData\");\n", "\n", "deltaTable\n", " .As(\"oldData\")\n", " .Merge(newData, \"oldData.id = newData.id\")\n", " .WhenMatched()\n", " .Update(new Dictionary() {{\"id\", Lit(\"-1\")}})\n", " .WhenNotMatched()\n", " .Insert(new Dictionary() {{\"id\", Col(\"newData.id\")}})\n", " .Execute();\n", "\n", "deltaTable.ToDF().Show(100);" ], "execution_count": 13 }, { "cell_type": "markdown", "source": [ "## History\n", "Delta's most powerful feature is the ability to allow looking into history i.e., the changes that were made to the underlying Delta Table. The cell below shows how simple it is to inspect the history." ] }, { "cell_type": "code", "source": [ "deltaTable.History().Show(20, 1000, false);" ], "execution_count": 14 }, { "cell_type": "markdown", "source": [ "## Read older versions of data using Time Travel\n", "\n", "You can query previous snapshots of your Delta Lake table by using a feature called Time Travel. If you want to access the data that you overwrote, you can query a snapshot of the table before you overwrote the first set of data using the versionAsOf option.\n", "\n", "Once you run the cell below, you should see the first set of data, from before you overwrote it. Time Travel is an extremely powerful feature that takes advantage of the power of the Delta Lake transaction log to access data that is no longer in the table. Removing the version 0 option (or specifying version 1) would let you see the newer data again. For more information, see [Query an older snapshot of a table (time travel)](https://docs.delta.io/latest/delta-batch.html#deltatimetravel)." ] }, { "cell_type": "code", "source": [ "var df = spark.Read().Format(\"delta\").Option(\"versionAsOf\", 0).Load(deltaTablePath);\n", "df.Show();" ], "execution_count": 15 }, { "cell_type": "markdown", "source": [ "## Write a stream of data to a table\n", "\n", "You can also write to a Delta Lake table using Spark's Structured Streaming. The Delta Lake transaction log guarantees exactly-once processing, even when there are other streams or batch queries running concurrently against the table. By default, streams run in append mode, which adds new records to the table.\n", "\n", "For more information about Delta Lake integration with Structured Streaming, see [Table Streaming Reads and Writes](https://docs.delta.io/latest/delta-streaming.html).\n", "\n", "In the cells below, here's what we are doing:\n", "\n", "1. *Cell 28* Setup a simple Spark Structured Streaming job to generate a sequence and make the job write into our Delta Table\n", "2. *Cell 30* Show the newly appended data\n", "3. *Cell 31* Inspect history\n", "4. *Cell 32* Stop the structured streaming job\n", "5. *Cell 33* Inspect history <-- You'll notice appends have stopped" ] }, { "cell_type": "code", "source": [ "var streamingDf = spark.ReadStream().Format(\"rate\").Load();\n", "var stream = streamingDf.SelectExpr(\"value as id\").WriteStream().Format(\"delta\").Option(\"checkpointLocation\", $\"/tmp/checkpoint-{sessionId}\").Start(deltaTablePath);" ], "execution_count": 16 }, { "cell_type": "markdown", "source": [ "## Read a stream of changes from a table\n", "\n", "While the stream is writing to the Delta Lake table, you can also read from that table as streaming source. For example, you can start another streaming query that prints all the changes made to the Delta Lake table." ] }, { "cell_type": "code", "source": [ "deltaTable.ToDF().Sort(Col(\"id\").Desc()).Show(100);" ], "execution_count": 17 }, { "cell_type": "code", "source": [ "deltaTable.History().Drop(\"userId\", \"userName\", \"job\", \"notebook\", \"clusterId\", \"isolationLevel\", \"isBlindAppend\").Show(20, 1000, false);" ], "execution_count": 18 }, { "cell_type": "code", "source": [ "stream.Stop();" ], "execution_count": 19 }, { "cell_type": "code", "source": [ "deltaTable.History().Drop(\"userId\", \"userName\", \"job\", \"notebook\", \"clusterId\", \"isolationLevel\", \"isBlindAppend\").Show(100, 1000, false);" ], "execution_count": 20 }, { "cell_type": "markdown", "source": [ "## Compaction\n", "\n", "If a Delta Table is growing too large, you can compact it by repartitioning into a smaller number of files.\n", "\n", "The option `dataChange = false` is an optimization that tells Delta Lake to do the repartition without marking the underlying data as \"modified\". This ensures that any other concurrent operations (such as streaming reads/writes) aren't negatively impacted.\n", "" ] }, { "cell_type": "code", "source": [ "int partitionCount = 2;\n", "\n", "spark.Read()\n", " .Format(\"delta\")\n", " .Load(deltaTablePath)\n", " .Repartition(partitionCount)\n", " .Write()\n", " .Option(\"dataChange\", \"false\")\n", " .Format(\"delta\")\n", " .Mode(\"overwrite\")\n", " .Save(deltaTablePath);" ], "execution_count": 21 }, { "cell_type": "markdown", "source": [ "## Convert Parquet to Delta\n", "You can do an in-place conversion from the Parquet format to Delta." ] }, { "cell_type": "code", "source": [ "var parquetPath = $\"/parquet/parquet-table-{sessionId}\";\n", "\n", "var data = spark.Range(0,5);\n", "data.Write().Parquet(parquetPath);\n", "\n", "// Confirm that the data isn't in the Delta format\n", "DeltaTable.IsDeltaTable(parquetPath)" ], "execution_count": 22 }, { "cell_type": "code", "source": [ "DeltaTable.ConvertToDelta(spark, $\"parquet.`{parquetPath}`\");\n", "\n", "//Confirm that the converted data is now in the Delta format\n", "DeltaTable.IsDeltaTable(parquetPath)" ], "execution_count": 23 }, { "cell_type": "markdown", "source": [ "## SQL Support\n", "Delta supports table utility commands through SQL. You can use SQL to:\n", "* Get a DeltaTable's history\n", "* Vacuum a DeltaTable\n", "* Convert a Parquet file to Delta\n", "" ] }, { "cell_type": "code", "source": [ "spark.Sql($\"DESCRIBE HISTORY delta.`{deltaTablePath}`\").Show();" ], "execution_count": 24 }, { "cell_type": "code", "source": [ "spark.Sql($\"VACUUM delta.`{deltaTablePath}`\").Show();" ], "execution_count": 25 }, { "cell_type": "code", "source": [ "var parquetId = (new Random()).Next(10000000);\n", "var parquetPath = $\"/parquet/parquet-table-{sessionId}-{parquetId}\";\n", "\n", "var data = spark.Range(0,5);\n", "data.Write().Parquet(parquetPath);\n", "\n", "// Confirm that the data isn't in the Delta format\n", "DeltaTable.IsDeltaTable(parquetPath);\n", "\n", "// Use SQL to convert the parquet table to Delta\n", "spark.Sql($\"CONVERT TO DELTA parquet.`{parquetPath}`\");\n", "\n", "DeltaTable.IsDeltaTable(parquetPath);" ], "execution_count": 26 } ] } } ================================================ FILE: Synapse/pipeline/01-Grandparent.json ================================================ { "name": "01-Grandparent", "properties": { "description": "procfwk grandparent pipeline used optionally to bootstrap any wider processes in your Data Factory that then calls the processing framework.", "activities": [ { "name": "procfwk", "description": "Call procfwk", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "02-Parent", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "BatchName": { "value": "@pipeline().parameters.BatchName", "type": "Expression" } } } } ], "parameters": { "BatchName": { "type": "string", "defaultValue": "NotUsed" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "procfwk", "Grandparent" ] } } ================================================ FILE: Synapse/pipeline/02-Parent.json ================================================ { "name": "02-Parent", "properties": { "description": "ADF.procfwk parent pipeline used to bootstrap the orchestration framework in perform the first level ForEach calls in sequence for the metadata stages.", "activities": [ { "name": "Get Stages", "description": "Returns a distinct list of execution stages within the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Set Execution Id", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[procfwk].[GetStages]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@variables('ExecutionId')", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference" }, "firstRowOnly": false } }, { "name": "Execute Stages", "description": "Top level ForEach to sequentially call all processing stages within the framework metadata. Items for iteration passed from the Get Stages lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Stages", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Stages').output.value", "type": "Expression" }, "isSequential": true, "activities": [ { "name": "Stage Executor", "description": "Call to the framework generic child pipeline for a given execution stage.", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Log Stage Preparing", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "03-Child", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "StageId": { "value": "@item().StageId", "type": "Expression" }, "ExecutionId": { "value": "@variables('ExecutionId')", "type": "Expression" } } } }, { "name": "Log Stage Preparing", "description": "Update the current execution table flagging all pipelines within the stage as preparing.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Check and Update Blockers", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogStagePreparing]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@variables('ExecutionId')", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check and Update Blockers", "description": "Used to double check and stop the next execution stage if failures and blockers have be incurred. This also depends on the failure handling property value which defines the stored procedure behaviour.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[CheckForBlockedPipelines]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@variables('ExecutionId')", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Execution Wrapper", "description": "Wrapper to reset and restart processing or create a completely new execution instance of the framework metadata.", "type": "Lookup", "dependsOn": [ { "activity": "Clean Up Previous Run", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[procfwk].[ExecutionWrapper]", "storedProcedureParameters": { "CallingOrchestratorName": { "type": "String", "value": { "value": "@pipeline().DataFactory", "type": "Expression" } }, "BatchName": { "type": "String", "value": { "value": "@pipeline().parameters.BatchName", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference" } } }, { "name": "Check Outcome and Update Logs", "description": "After a successful execution run the current execution metadata is moved to the long term logging table by this stored procedure call. Otherwise an error will be raised.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Stages", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[UpdateExecutionLog]", "storedProcedureParameters": { "PerformErrorCheck": { "value": { "value": "@bool(1)", "type": "Expression" }, "type": "Boolean" }, "ExecutionId": { "value": { "value": "@variables('ExecutionId')", "type": "Expression" }, "type": "Guid" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check Previous Execution", "description": "Query the current execution table for worker pipelines that require a clean up from the previous execution run.", "type": "Lookup", "dependsOn": [ { "activity": "Execute Precursor", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[procfwk].[CheckPreviousExeuction]", "storedProcedureParameters": { "BatchName": { "type": "String", "value": { "value": "@pipeline().parameters.BatchName", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference" }, "firstRowOnly": false } }, { "name": "Clean Up Previous Run", "description": "Handle Worker pipelines that are reported as Running when the parent pipeline is called again. Get what the actual status of those pipelines is.", "type": "ForEach", "dependsOn": [ { "activity": "Check Previous Execution", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Check Metadata Integrity", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Check Previous Execution').output.value", "type": "Expression" }, "isSequential": false, "batchCount": 50, "activities": [ { "name": "Get SPN Details", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": true, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[procfwk].[GetWorkerAuthDetails]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@item().LocalExecutionId", "type": "Expression" } }, "PipelineId": { "type": "Int32", "value": { "value": "@item().PipelineId", "type": "Expression" } }, "StageId": { "type": "Int32", "value": { "value": "@item().StageId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference" } } }, { "name": "Log Pipeline Checking", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineChecking]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Pipeline Status", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Get SPN Details", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Log Pipeline Checking", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "CheckPipelineStatus", "method": "POST", "body": { "value": "@concat('\n{\n \"tenantId\": \"',activity('Get SPN Details').output.firstRow.TenantId,'\",\n \"applicationId\": \"',activity('Get SPN Details').output.firstRow.AppId,'\",\n \"authenticationKey\": \"',activity('Get SPN Details').output.firstRow.AppSecret,'\",\n \"subscriptionId\": \"',activity('Get SPN Details').output.firstRow.SubscriptionId,'\",\n \"resourceGroupName\": \"',item().ResourceGroupName,'\",\n \"orchestratorName\": \"',item().OrchestratorName,'\",\n \"orchestratorType\": \"',item().OrchestratorType,'\",\n \"pipelineName\": \"',item().PipelineName,'\",\n \"runId\": \"',item().PipelineRunId,'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Set Pipeline Status", "description": "Update the metadata depending on the actual pipeline outcome. Using the status as the case.", "type": "Switch", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@activity('Get Pipeline Status').output.ActualStatus", "type": "Expression" }, "cases": [ { "value": "Failed", "activities": [ { "name": "Pipeline Status Failed", "description": "Updates the current execution table with a pipeline status of failed if the function outcome is failed. Also blocks pipelines in the downstream execution stage.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": null, "type": "Guid" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Succeeded", "activities": [ { "name": "Pipeline Status Succeeded", "description": "Updates the current execution table with a pipeline status of success if the function outcome is succeeded.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineSuccess]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Queued", "activities": [ { "name": "Pipeline Status Queued - Running", "description": "Updates the current execution table with a pipeline status of running if the function outcome is queued.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "InProgress", "activities": [ { "name": "Pipeline Status InProgress - Running", "description": "Updates the current execution table with a pipeline status of running if the function outcome is in progress.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Cancelled", "activities": [ { "name": "Pipeline Status Cancelled", "description": "Updates the current execution table with a pipeline status of cancelled if the function outcome is cancelled.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineCancelled]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" }, "CleanUpRun": { "value": { "value": "@bool(1)", "type": "Expression" }, "type": "Boolean" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } ], "defaultActivities": [ { "name": "Pipeline Status Unknown", "description": "Updates the current execution table with a pipeline status of unknown if the function returns an unexpected outcome.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineUnknown]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" }, "CleanUpRun": { "value": { "value": "@bool(1)", "type": "Expression" }, "type": "Boolean" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Set Last Check DateTime", "description": "Update the current execution table with a date time from when the function last checked the pipeline status.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineLastStatusCheck]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@item().LocalExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@item().PipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@item().StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Execute Precursor", "description": "Uses the database property value ExecutionPrecursorProc to run any custom logic against the metadata database before the execution run starts.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[ExecutePrecursorProcedure]" }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Set Execution Id", "description": "Set the local execution Id to a pipeline variable for each in several downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Execution Wrapper", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "ExecutionId", "value": { "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", "type": "Expression" } } }, { "name": "Check Metadata Integrity", "description": "Performs a series of checks on all metadata held in the framework SQLDB. This is intended to raise errors before an execution run even starts.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Precursor", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[CheckMetadataIntegrity]", "storedProcedureParameters": { "BatchName": { "value": { "value": "@pipeline().parameters.BatchName", "type": "Expression" }, "type": "String" }, "DebugMode": { "value": { "value": "@bool(0)", "type": "Expression" }, "type": "Boolean" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ], "parameters": { "BatchName": { "type": "string", "defaultValue": "NotUsed" } }, "variables": { "ExecutionId": { "type": "String" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "procfwk", "Parent" ] } } ================================================ FILE: Synapse/pipeline/03-Child.json ================================================ { "name": "03-Child", "properties": { "description": "procfwk child pipeline used to execute Worker pipelines within a given execution stage. This pipeline will be called once for each stage, then execute all Workers in parallel.", "activities": [ { "name": "Get Pipelines", "description": "Returns all pipelines from the metadata to be executed within a given processing stage.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[procfwk].[GetPipelinesInStage]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" } }, "StageId": { "type": "Int32", "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference" }, "firstRowOnly": false } }, { "name": "Execute Pipelines", "description": "Second level ForEach to run in parallel all pipelines within the stage. Items for iteration passed from the Get Pipelines lookup activity.", "type": "ForEach", "dependsOn": [ { "activity": "Get Pipelines", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Pipelines').output.value", "type": "Expression" }, "isSequential": false, "batchCount": 50, "activities": [ { "name": "Worker Pipeline Executor", "description": "Run the required worker pipeline and wait for its completion. Update metadata once done.", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "04-Infant", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "executionId": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "stageId": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "pipelineId": { "value": "@item().PipelineId", "type": "Expression" } } } } ] } } ], "parameters": { "StageId": { "type": "int" }, "ExecutionId": { "type": "string" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "procfwk", "Child" ] } } ================================================ FILE: Synapse/pipeline/04-Infant.json ================================================ { "name": "04-Infant", "properties": { "description": "procfwk infant pipeline used to check when the processing pipeline called by the Child completes and passes the resulting status back to the metadata database.", "activities": [ { "name": "Execute Worker Pipeline", "description": "The lowest level executor with the metadata framework to call existing processing pipelines within Data Factory. The function called will block processing and wait for an outcome.", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Log Pipeline Running", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Get Pipeline Params", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "ExecutePipeline", "method": "POST", "body": { "value": "@concat('\n{\n \"tenantId\": \"',variables('WorkerCoreDetails')[0].tenantId,'\",\n \"applicationId\": \"',variables('WorkerCoreDetails')[0].applicationId,'\",\n \"authenticationKey\": \"',variables('WorkerCoreDetails')[0].authenticationKey,'\",\n \"subscriptionId\": \"',variables('WorkerCoreDetails')[0].subscriptionId,'\",\n \"resourceGroupName\": \"',variables('WorkerCoreDetails')[0].resourceGroupName,'\",\n\t\"orchestratorName\": \"',variables('WorkerCoreDetails')[0].orchestratorName,'\",\n \"orchestratorType\": \"',variables('WorkerCoreDetails')[0].orchestratorType,'\",\n \"pipelineName\": \"',variables('WorkerCoreDetails')[0].pipelineName,'\"',activity('Get Pipeline Params').output.firstRow.Params,'\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Get Pipeline Params", "description": "Returns any parameters from metadata required for the processing pipeline being called. The output can be an empty string if no parameters are required.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[procfwk].[GetPipelineParameters]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference" } } }, { "name": "Log Pipeline Running", "description": "Sets the current pipeline with a status of running within the current execution database table.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Is Target Worker Validate", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineRunning]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Log Execute Function Activity Failure", "description": "Handle true failures from calling out to the Azure Function and update the current execution table accordingly so a restart can occur.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Execute Worker Pipeline", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" }, "CallingActivity": { "value": "ExecuteWorkerPipeline", "type": "String" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Update Run Id", "description": "Provide the actual ADF run ID back to the current execution table for long term logging and alignment between the metadata other Azure monitoring tools.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Set Run Id", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineRunId]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@variables('WorkerRunId')", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Check For Alerts", "description": "Checks the properties tables and if any recipients in the database require alerts sending for the current pipeline ID.", "type": "Lookup", "dependsOn": [ { "activity": "Update Run Id", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Pipeline Result", "dependencyConditions": [ "Completed" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[procfwk].[CheckForEmailAlerts]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference" }, "firstRowOnly": true } }, { "name": "Send Alerts", "description": "True = alerts need sending.\nFalse = do nothing.", "type": "IfCondition", "dependsOn": [ { "activity": "Check For Alerts", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@activity('Check For Alerts').output.firstRow.SendAlerts", "type": "Expression" }, "ifTrueActivities": [ { "name": "Get Email Parts", "description": "Return all required content from the metadata database to send an email alerting using the procfwk. The lookup returns the exact content for the function body request.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": true, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[procfwk].[GetEmailAlertParts]", "storedProcedureParameters": { "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference" }, "firstRowOnly": true } }, { "name": "Call Email Sender", "description": "Pass off email request to Utils Send Email pipeline.", "type": "ExecutePipeline", "dependsOn": [ { "activity": "Get Email Parts", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "Email Sender", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "Recipients": { "value": "@activity('Get Email Parts').output.firstRow.emailRecipients", "type": "Expression" }, "CcRecipients": { "value": "@activity('Get Email Parts').output.firstRow.emailCcRecipients", "type": "Expression" }, "BccRecipients": { "value": "@activity('Get Email Parts').output.firstRow.emailBccRecipients", "type": "Expression" }, "Subject": { "value": "@activity('Get Email Parts').output.firstRow.emailSubject", "type": "Expression" }, "Body": { "value": "@activity('Get Email Parts').output.firstRow.emailBody", "type": "Expression" }, "Importance": { "value": "@activity('Get Email Parts').output.firstRow.emailImportance", "type": "Expression" } } } } ] } }, { "name": "Wait Until Pipeline Completes", "description": "Loops until the Worker pipeline called completes.\n\nSimple status:\n- Running = new iteration.\n- Done = break.", "type": "Until", "dependsOn": [ { "activity": "Get Wait Duration", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Execute Worker Pipeline", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Run Id", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@variables('WorkerPipelineState')", "type": "Expression" }, "activities": [ { "name": "Get Worker Pipeline Status", "description": "Checks the status of a given processing pipeline and provides the value for the downstream framework activities to act upon.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "CheckPipelineStatus", "method": "POST", "body": { "value": "@concat('\n{\n \"tenantId\": \"',variables('WorkerCoreDetails')[0].tenantId,'\",\n \"applicationId\": \"',variables('WorkerCoreDetails')[0].applicationId,'\",\n \"authenticationKey\": \"',variables('WorkerCoreDetails')[0].authenticationKey,'\",\n \"subscriptionId\": \"',variables('WorkerCoreDetails')[0].subscriptionId,'\",\n \"resourceGroupName\": \"',variables('WorkerCoreDetails')[0].resourceGroupName,'\",\n\t\"orchestratorName\": \"',variables('WorkerCoreDetails')[0].orchestratorName,'\",\n \"orchestratorType\": \"',variables('WorkerCoreDetails')[0].orchestratorType,'\",\n \"pipelineName\": \"',variables('WorkerCoreDetails')[0].pipelineName,'\",\n \"runId\": \"',variables('WorkerRunId'),'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Wait If Running", "description": "True = Do nothing.\nFalse = Wait, before the next iteration.", "type": "IfCondition", "dependsOn": [ { "activity": "Set Worker State", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@variables('WorkerPipelineState')", "type": "Expression" }, "ifFalseActivities": [ { "name": "Wait for Pipeline", "description": "The processing pipeline is still running so Wait before checking its status again.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@activity('Get Wait Duration').output.firstRow.PropertyValue", "type": "Expression" } } } ] } }, { "name": "Set Last Check DateTime", "description": "Update the current execution table with a date time from when the Worker pipeline status was last checked as part of the Until iterations.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Worker Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineLastStatusCheck]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Log Check Function Activity Failure", "description": "Report to the current execution table that the framework pipeline activity has failed. This failure is outside of the scope of the framework and is probably related to a wider platform problem.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Worker Pipeline Status", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "CallingActivity": { "value": "GetWorkerPipelineStatus", "type": "String" }, "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Set Worker State", "description": "Set the bool state of the Worker pipeline to be used by the Until and If expressions. True = Complete, False = Running.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Pipeline Status", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerPipelineState", "value": { "value": "@equals('Complete',activity('Get Worker Pipeline Status').output.SimpleStatus)", "type": "Expression" } } } ], "timeout": "0.00:10:00" } }, { "name": "Set Pipeline Result", "description": "Receives the outcome from the function execution for a given processing pipeline and updates the current execution table with different pipelines status values depending on the result (case).", "type": "Switch", "dependsOn": [ { "activity": "Wait Until Pipeline Completes", "dependencyConditions": [ "Completed" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@activity('Get Worker Pipeline Status').output.ActualStatus", "type": "Expression" }, "cases": [ { "value": "Succeeded", "activities": [ { "name": "Pipeline Status Succeeded", "description": "Updates the current execution table with a pipeline status of success if the function outcome is succeeded.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineSuccess]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Failed", "activities": [ { "name": "Pipeline Status Failed", "description": "Updates the current execution table with a pipeline status of failed if the function outcome is failed. Also blocks pipelines in the downstream execution stage.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "RunId": { "value": { "value": "@variables('WorkerRunId')", "type": "Expression" }, "type": "Guid" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Worker Pipeline Error Details", "description": "Get the activity error details for the run ID of the worker pipeline called. Returns an array of all errors.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "GetActivityErrors", "method": "POST", "body": { "value": "@concat('\n{\n \"tenantId\": \"',variables('WorkerCoreDetails')[0].tenantId,'\",\n \"applicationId\": \"',variables('WorkerCoreDetails')[0].applicationId,'\",\n \"authenticationKey\": \"',variables('WorkerCoreDetails')[0].authenticationKey,'\",\n \"subscriptionId\": \"',variables('WorkerCoreDetails')[0].subscriptionId,'\",\n \"resourceGroupName\": \"',variables('WorkerCoreDetails')[0].resourceGroupName,'\",\n\t\"orchestratorName\": \"',variables('WorkerCoreDetails')[0].orchestratorName,'\",\n \"orchestratorType\": \"',variables('WorkerCoreDetails')[0].orchestratorType,'\",\n \"pipelineName\": \"',variables('WorkerCoreDetails')[0].pipelineName,'\",\n \"runId\": \"',variables('WorkerRunId'),'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Log Error Details", "description": "Parses pipeline error details and persists them to the metadata database error log table.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Get Worker Pipeline Error Details", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetErrorLogDetails]", "storedProcedureParameters": { "JsonErrorDetails": { "value": { "value": "@string(activity('Get Worker Pipeline Error Details').output)", "type": "Expression" }, "type": "String" }, "LocalExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] }, { "value": "Cancelled", "activities": [ { "name": "Pipeline Status Cancelled", "description": "Updates the current execution table with a pipeline status of cancelled if the function outcome is cancelled.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineCancelled]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } ], "defaultActivities": [ { "name": "Pipeline Status Unknown", "description": "Updates the current execution table with a pipeline status of unknown if the function returns an unexpected outcome.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineUnknown]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Get Wait Duration", "description": "Return wait duration in seconds from database properties table to be used during each Until iteration when the Worker pipeline is still running.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "PipelineStatusCheckDuration" } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference" } } }, { "name": "Set Run Id", "description": "Set local variable from activity output once for value reuse in downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Execute Worker Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerRunId", "value": { "value": "@activity('Execute Worker Pipeline').output.RunId", "type": "Expression" } } }, { "name": "Validate Pipeline", "description": "Query the target data factory and establish if the provided worker pipeline name is valid.", "type": "AzureFunctionActivity", "dependsOn": [ { "activity": "Log Pipeline Validating", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Capture Worker Core Details as an Array", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": true }, "userProperties": [], "typeProperties": { "functionName": "ValidatePipeline", "method": "POST", "body": { "value": "@concat('\n{\n \"tenantId\": \"',variables('WorkerCoreDetails')[0].tenantId,'\",\n \"applicationId\": \"',variables('WorkerCoreDetails')[0].applicationId,'\",\n \"authenticationKey\": \"',variables('WorkerCoreDetails')[0].authenticationKey,'\",\n \"subscriptionId\": \"',variables('WorkerCoreDetails')[0].subscriptionId,'\",\n \"resourceGroupName\": \"',variables('WorkerCoreDetails')[0].resourceGroupName,'\",\n\t\"orchestratorName\": \"',variables('WorkerCoreDetails')[0].orchestratorName,'\",\n \"orchestratorType\": \"',variables('WorkerCoreDetails')[0].orchestratorType,'\",\n \"pipelineName\": \"',variables('WorkerCoreDetails')[0].pipelineName,'\"\n}')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } }, { "name": "Is Target Worker Validate", "description": "True = the worker pipeline name is valid.\nFalse = the worker pipeline name is invalid. Raise an exception.", "type": "IfCondition", "dependsOn": [ { "activity": "Validate Pipeline", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@bool(activity('Validate Pipeline').output.PipelineExists)", "type": "Expression" }, "ifFalseActivities": [ { "name": "Throw Exception - Invalid Infant", "description": "Throw an exception with details about the invalid worker pipeline name.", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "Throw Exception", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "Message": { "value": "@concat('Worker pipeline [',variables('WorkerCoreDetails')[0].pipelineName,'] is not valid in target Orchestrator [',variables('WorkerCoreDetails')[0].orchestratorName,']')", "type": "Expression" } } } }, { "name": "Update Execution With Invalid Worker", "description": "Update the current execution table with an informed status for the worker pipeline that couldn't be executed.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "CallingActivity": { "value": "InvalidPipelineName", "type": "String" }, "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } }, { "name": "Log Validate Function Activity Failure", "description": "Handle true failures from calling out to the Azure Function and update the current execution table accordingly so a restart can occur.", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Validate Pipeline", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogActivityFailed]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" }, "CallingActivity": { "value": "ValidatePipeline", "type": "String" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Log Pipeline Validating", "description": "Sets the current pipeline with a status of validating within the current execution database table.", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[procfwk].[SetLogPipelineValidating]", "storedProcedureParameters": { "ExecutionId": { "value": { "value": "@pipeline().parameters.ExecutionId", "type": "Expression" }, "type": "Guid" }, "PipelineId": { "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" }, "type": "Int32" }, "StageId": { "value": { "value": "@pipeline().parameters.StageId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } }, { "name": "Get Worker Core Details", "description": "Return worker pipeline information for metadata database. Including target data factory, pipeline name and resource group. Return the SPN ID and Secret for the worker pipeline being executed. Called at this level as each pipeline can have a different SPN.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": true, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[procfwk].[GetWorkerDetailsWrapper]", "storedProcedureParameters": { "ExecutionId": { "type": "Guid", "value": { "value": "@pipeline().parameters.executionId", "type": "Expression" } }, "PipelineId": { "type": "Int32", "value": { "value": "@pipeline().parameters.pipelineId", "type": "Expression" } }, "StageId": { "type": "Int32", "value": { "value": "@pipeline().parameters.stageId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference" } } }, { "name": "Capture Worker Core Details as an Array", "description": "Add all worker pipeline details to a local variable array that can be accessed by each function call requiring the values.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Worker Core Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerCoreDetails", "value": { "value": "@array(activity('Get Worker Core Details').output.firstRow)", "type": "Expression" } } } ], "parameters": { "executionId": { "type": "string" }, "stageId": { "type": "int" }, "pipelineId": { "type": "int" } }, "variables": { "WorkerPipelineState": { "type": "Boolean" }, "WorkerRunId": { "type": "String" }, "WorkerCoreDetails": { "type": "Array" } }, "folder": { "name": "_ProcFwk" }, "annotations": [ "procfwk", "Infant" ] } } ================================================ FILE: Synapse/pipeline/Check For Running Pipeline.json ================================================ { "name": "Check For Running Pipeline", "properties": { "description": "For a given pipeline and optional batch name establish if a pipeline run is already in progress. Throw an exception if it it.", "activities": [ { "name": "Filter Running Pipelines", "description": "Filter the pipeline runs results for pipelines that exclude the current triggered run and that are currently running (in progress or queued).", "type": "Filter", "dependsOn": [ { "activity": "Switch For Orchestrator Type", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@variables('PipelineRuns')", "type": "Expression" }, "condition": { "value": "@and(not(equals(item().runId,pipeline().parameters.ThisRunId)),or(equals(item().status,'InProgress'),equals(item().status,'Queued')))", "type": "Expression" } } }, { "name": "Get Framework Orchestrator Details", "description": "Using the metadata orchestrators return details about the resource running the framework pipelines.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[procfwk].[GetFrameworkOrchestratorDetails]", "storedProcedureParameters": { "CallingOrchestratorName": { "type": "String", "value": { "value": "@pipeline().DataFactory", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference" } } }, { "name": "Get Query Run Days Value", "description": "Using the metadata properties table return the run days value to provide the API request with a date range for pipeline executions.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "PreviousPipelineRunsQueryRange" } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference" } } }, { "name": "If Pipeline Is Running", "description": "If the running pipeline count is greater than or equal to one.\nTrue = raise an exception.", "type": "IfCondition", "dependsOn": [ { "activity": "If Using Batch Executions", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@greaterOrEquals(int(variables('RunCount')),1)", "type": "Expression" }, "ifTrueActivities": [ { "name": "Throw Exception - Pipeline Running", "description": "Using the utils pipeline raise an exception to stop the new trigger while a run is already in progress.", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "Throw Exception", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "Message": { "value": "@concat('Provided pipeline name (',pipeline().parameters.PipelineName,') still has a run in progress or queued given the query range parameters set in the properties table.')", "type": "Expression" } } } } ] } }, { "name": "Get Execution Batch Status", "description": "Using the metadata properties table return the flag to indicate if batch execution setting are enabled or disabled.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[procfwk].[GetPropertyValue]", "storedProcedureParameters": { "PropertyName": { "type": "String", "value": "UseExecutionBatches" } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference" } } }, { "name": "If Using Batch Executions", "description": "True = batch executions are enabled.\nFalse = batch execution are disabled.", "type": "IfCondition", "dependsOn": [ { "activity": "Get Execution Batch Status", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Filter Running Pipelines", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@equals(activity('Get Execution Batch Status').output.firstRow.PropertyValue,string(1))", "type": "Expression" }, "ifFalseActivities": [ { "name": "Set Run Count Without Batch", "description": "Set the pipelines running count variable to be tested later.", "type": "SetVariable", "dependsOn": [], "userProperties": [], "typeProperties": { "variableName": "RunCount", "value": { "value": "@string(activity('Filter Running Pipelines').output.FilteredItemsCount)", "type": "Expression" } } } ], "ifTrueActivities": [ { "name": "Filter for Batch Name", "description": "Further filter the return pipeline runs for any running pipelines with the same batch name value.", "type": "Filter", "dependsOn": [], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Filter Running Pipelines').output.value", "type": "Expression" }, "condition": { "value": "@equals(item().parameters.BatchName,pipeline().parameters.BatchName)", "type": "Expression" } } }, { "name": "Set Run Count for Batch", "description": "Set the resulting pipeline running count variable to be tested later.", "type": "SetVariable", "dependsOn": [ { "activity": "Filter for Batch Name", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "RunCount", "value": { "value": "@string(activity('Filter for Batch Name').output.FilteredItemsCount)", "type": "Expression" } } } ] } }, { "name": "Set Subscription Id", "description": "Set the subscription Id value to a local variable for use in various downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Framework Orchestrator Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "SubscriptionId", "value": { "value": "@activity('Get Framework Orchestrator Details').output.firstRow.SubscriptionId", "type": "Expression" } } }, { "name": "Set Resource Group Name", "description": "Set the resource group name value to a local variable for use in various downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Framework Orchestrator Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "ResourceGroupName", "value": { "value": "@activity('Get Framework Orchestrator Details').output.firstRow.ResourceGroupName", "type": "Expression" } } }, { "name": "Set Orchestrator Type", "description": "Set the orchestrator type value to a local variable for use in various downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Framework Orchestrator Details", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "OrchestratorType", "value": { "value": "@toUpper(activity('Get Framework Orchestrator Details').output.firstRow.OrchestratorType)", "type": "Expression" } } }, { "name": "Switch For Orchestrator Type", "description": "Switch and handle requests for both Azure Data Factory (ADF) and Azure Synapse Analytics (SYN).", "type": "Switch", "dependsOn": [ { "activity": "Set Orchestrator Type", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Query Run Days", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Resource Group Name", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Set Subscription Id", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "on": { "value": "@variables('OrchestratorType')", "type": "Expression" }, "cases": [ { "value": "ADF", "activities": [ { "name": "Check for Valid ADF Pipeline Name", "description": "Use the Azure Management API to return and establish if the framework pipeline exists in the target Data Factory instance, including being deployed.", "type": "WebActivity", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "url": { "value": "https://management.azure.com/subscriptions/@{variables('SubscriptionId')}/resourceGroups/@{variables('ResourceGroupName')}/providers/Microsoft.DataFactory/factories/@{pipeline().DataFactory}/pipelines/@{pipeline().parameters.PipelineName}?api-version=2018-06-01", "type": "Expression" }, "method": "GET", "authentication": { "type": "MSI", "resource": "https://management.core.windows.net/" } } }, { "name": "Get ADF Pipeline Runs", "description": "Use the Azure Management API to return a list of data factory pipeline runs within the given time window.", "type": "WebActivity", "dependsOn": [ { "activity": "Check for Valid ADF Pipeline Name", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "url": { "value": "https://management.azure.com/subscriptions/@{variables('SubscriptionId')}/resourceGroups/@{variables('ResourceGroupName')}/providers/Microsoft.DataFactory/factories/@{pipeline().DataFactory}/queryPipelineRuns?api-version=2018-06-01", "type": "Expression" }, "method": "POST", "body": { "value": "{\n \"lastUpdatedAfter\": \"@{adddays(utcnow(),int(variables('QueryRunDays')))}\",\n \"lastUpdatedBefore\": \"@{utcnow()}\",\n \"filters\": [\n {\n \"operand\": \"PipelineName\",\n \"operator\": \"Equals\",\n \"values\": [\n \"@{pipeline().parameters.PipelineName}\"\n ]\n }\n ]\n}", "type": "Expression" }, "authentication": { "type": "MSI", "resource": "https://management.core.windows.net/" } } }, { "name": "Set ADF Runs Output", "description": "Set output to local array for use in downstream filtering and pipeline checks. Use the same array output for both switch cases.", "type": "SetVariable", "dependsOn": [ { "activity": "Get ADF Pipeline Runs", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "PipelineRuns", "value": { "value": "@activity('Get ADF Pipeline Runs').output.value", "type": "Expression" } } } ] }, { "value": "SYN", "activities": [ { "name": "Check for Valid SYN Pipeline Name", "description": "Use the Azure Management API to return and establish if the framework pipeline exists in the target Synapse instance, including being deployed.\n\nSee: https://docs.microsoft.com/en-us/rest/api/synapse/data-plane/pipeline/getpipeline", "type": "WebActivity", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "url": { "value": "https://@{pipeline().DataFactory}.dev.azuresynapse.net/pipelines/@{pipeline().parameters.PipelineName}?api-version=2019-06-01-preview", "type": "Expression" }, "method": "GET", "authentication": { "type": "MSI", "resource": "https://management.core.windows.net/" } } }, { "name": "Get SYN Pipeline Runs", "description": "Use the Azure Management API to return a list of synapse pipeline runs within the given time window.\n\nSee: https://docs.microsoft.com/en-us/rest/api/synapse/data-plane/pipelinerun/querypipelinerunsbyworkspace", "type": "WebActivity", "dependsOn": [ { "activity": "Check for Valid SYN Pipeline Name", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "url": { "value": "https://@{pipeline().DataFactory}.dev.azuresynapse.net/queryPipelineRuns?api-version=2019-06-01-preview", "type": "Expression" }, "method": "POST", "body": { "value": "{\n \"lastUpdatedAfter\": \"@{adddays(utcnow(),int(variables('QueryRunDays')))}\",\n \"lastUpdatedBefore\": \"@{utcnow()}\",\n \"filters\": [\n {\n \"operand\": \"PipelineName\",\n \"operator\": \"Equals\",\n \"values\": [\n \"@{pipeline().parameters.PipelineName}\"\n ]\n }\n ]\n}", "type": "Expression" }, "authentication": { "type": "MSI", "resource": "https://management.core.windows.net/" } } }, { "name": "Set SYN Runs Output", "description": "Set output to local array for use in downstream filtering and pipeline checks. Use the same array output for both switch cases.", "type": "SetVariable", "dependsOn": [ { "activity": "Get SYN Pipeline Runs", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "PipelineRuns", "value": { "value": "@activity('Get SYN Pipeline Runs').output.value", "type": "Expression" } } } ] } ], "defaultActivities": [ { "name": "Throw Exception Invalid Orchestrator Type", "description": "Throw exception if switch cases are not met.", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "Throw Exception", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "Message": "Invalid orchestrator type provided. Unable to check pipeline running state." } } } ] } }, { "name": "Set Query Run Days", "description": "Set the query run days value to a local variable for use in various downstream activities.", "type": "SetVariable", "dependsOn": [ { "activity": "Get Query Run Days Value", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "QueryRunDays", "value": { "value": "@activity('Get Query Run Days Value').output.firstRow.PropertyValue", "type": "Expression" } } } ], "parameters": { "BatchName": { "type": "string", "defaultValue": "NotUsed" }, "PipelineName": { "type": "string" }, "ThisRunId": { "type": "string" } }, "variables": { "SubscriptionId": { "type": "String" }, "RunCount": { "type": "String" }, "ResourceGroupName": { "type": "String" }, "OrchestratorType": { "type": "String" }, "QueryRunDays": { "type": "String" }, "PipelineRuns": { "type": "Array" } }, "folder": { "name": "_ProcFwk/_ProcFwkUtils" }, "annotations": [ "procfwk", "Utils" ] } } ================================================ FILE: Synapse/pipeline/Email Sender.json ================================================ { "name": "Email Sender", "properties": { "description": "Provide a simple abstract over the send email function with request body item exposed as pipeline parameters.", "activities": [ { "name": "Send Email", "description": "Use an Azure Function to perform an SMTP client email send operation.", "type": "AzureFunctionActivity", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "functionName": "SendEmail", "method": "POST", "body": { "value": "{\n\"emailRecipients\": \"@{pipeline().parameters.Recipients}\",\n\"emailCcRecipients\": \"@{pipeline().parameters.CcRecipients}\",\n\"emailBccRecipients\": \"@{pipeline().parameters.BccRecipients}\",\n\"emailSubject\": \"@{pipeline().parameters.Subject}\",\n\"emailBody\": \"@{pipeline().parameters.Body}\",\n\"emailImportance\": \"@{pipeline().parameters.Importance}\"\n}", "type": "Expression" } }, "linkedServiceName": { "referenceName": "FrameworkFunctions", "type": "LinkedServiceReference" } } ], "parameters": { "Recipients": { "type": "string" }, "CcRecipients": { "type": "string" }, "BccRecipients": { "type": "string" }, "Subject": { "type": "string" }, "Body": { "type": "string" }, "Importance": { "type": "string" } }, "folder": { "name": "_ProcFwk/_ProcFwkUtils" }, "annotations": [ "procfwk", "Utils" ] } } ================================================ FILE: Synapse/pipeline/Intentional Error.json ================================================ { "name": "Intentional Error", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait1", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } }, { "name": "Raise Errors or Not", "description": "Framework development worker simulator.", "type": "IfCondition", "dependsOn": [ { "activity": "Wait1", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "expression": { "value": "@equals(pipeline().parameters.RaiseErrors,'true')", "type": "Expression" }, "ifTrueActivities": [ { "name": "Call Fail Procedure", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[dbo].[FailProcedure]", "storedProcedureParameters": { "RaiseError": { "value": { "value": "@pipeline().parameters.RaiseErrors", "type": "Expression" }, "type": "String" } } }, "linkedServiceName": { "referenceName": "SupportDatabase", "type": "LinkedServiceReference" } } ] } } ], "parameters": { "RaiseErrors": { "type": "string", "defaultValue": "false" }, "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] } } ================================================ FILE: Synapse/pipeline/Throw Exception.json ================================================ { "name": "Throw Exception", "properties": { "description": "Provide a simple way of throwing an exception within Data Factory using TSQL error handling.", "activities": [ { "name": "Raise Error", "description": "Using a SQL database to raise an error/exception but wrapped up as a data factory pipeline. Error message information exposed as a pipeline parameter.", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.00:10:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderQuery": { "value": "RAISERROR('@{pipeline().parameters.Message}',16,1);", "type": "Expression" }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference" }, "firstRowOnly": false } } ], "parameters": { "Message": { "type": "string" } }, "folder": { "name": "_ProcFwk/_ProcFwkUtils" }, "annotations": [ "procfwk", "Utils" ] } } ================================================ FILE: Synapse/pipeline/Wait 1.json ================================================ { "name": "Wait 1", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait1", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] } } ================================================ FILE: Synapse/pipeline/Wait 10.json ================================================ { "name": "Wait 10", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait10", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] } } ================================================ FILE: Synapse/pipeline/Wait 2.json ================================================ { "name": "Wait 2", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait2", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] } } ================================================ FILE: Synapse/pipeline/Wait 3.json ================================================ { "name": "Wait 3", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait3", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] } } ================================================ FILE: Synapse/pipeline/Wait 4.json ================================================ { "name": "Wait 4", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait4", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] } } ================================================ FILE: Synapse/pipeline/Wait 5.json ================================================ { "name": "Wait 5", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait5", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] } } ================================================ FILE: Synapse/pipeline/Wait 6.json ================================================ { "name": "Wait 6", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait6", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] } } ================================================ FILE: Synapse/pipeline/Wait 7.json ================================================ { "name": "Wait 7", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait7", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] } } ================================================ FILE: Synapse/pipeline/Wait 8.json ================================================ { "name": "Wait 8", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait8", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 5 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] } } ================================================ FILE: Synapse/pipeline/Wait 9.json ================================================ { "name": "Wait 9", "properties": { "description": "Used just so the procfwk has something to call during development.", "activities": [ { "name": "Wait9", "description": "Framework development worker simulator.", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": { "value": "@pipeline().parameters.WaitTime", "type": "Expression" } } } ], "parameters": { "WaitTime": { "type": "int", "defaultValue": 15 } }, "folder": { "name": "_Workers" }, "annotations": [ "_ProcFwkWorker" ] } } ================================================ FILE: Synapse/trigger/FunctionalTestingTrigger.json ================================================ { "name": "FunctionalTestingTrigger", "properties": { "description": "Used for functional testing of the framework in a dedicated environment.", "annotations": [ "procfwk" ], "runtimeState": "Stopped", "pipelines": [ { "pipelineReference": { "referenceName": "01-Grandparent", "type": "PipelineReference" } } ], "type": "ScheduleTrigger", "typeProperties": { "recurrence": { "frequency": "Hour", "interval": 2, "startTime": "2020-04-06T15:00:00Z", "timeZone": "UTC" } } } }